diff --git a/swh/loader/svn/tests/conftest.py b/swh/loader/svn/tests/conftest.py --- a/swh/loader/svn/tests/conftest.py +++ b/swh/loader/svn/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019-2021 The Software Heritage developers +# Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -7,6 +7,8 @@ import pytest +from .utils import create_repo + @pytest.fixture def swh_storage_backend_config(swh_storage_backend_config): @@ -37,3 +39,9 @@ "check_revision": 100, "temp_directory": "/tmp", } + + +@pytest.fixture +def repo_url(tmpdir_factory): + # create a repository + return create_repo(tmpdir_factory.mktemp("repos")) diff --git a/swh/loader/svn/tests/test_externals.py b/swh/loader/svn/tests/test_externals.py new file mode 100644 --- /dev/null +++ b/swh/loader/svn/tests/test_externals.py @@ -0,0 +1,1267 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from swh.loader.svn.loader import SvnLoader, SvnLoaderFromRemoteDump +from swh.loader.svn.utils import svn_urljoin +from swh.loader.tests import assert_last_visit_matches, check_snapshot + +from .utils import CommitChange, CommitChangeType, add_commit, create_repo + + +@pytest.fixture +def external_repo_url(tmpdir_factory): + # create a repository + return create_repo(tmpdir_factory.mktemp("external")) + + +def test_loader_with_valid_svn_externals( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create some directories and files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/hello/hello-world", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho Hello World !", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="foo.sh", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Create repository structure.", + [ + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="branches/",), + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="tags/",), + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/bar.sh", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + # second commit + add_commit( + repo_url, + ( + "Set svn:externals property on trunk/externals path of repository to load." + "One external targets a remote directory and another one a remote file." + ), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/hello')} hello\n" + f"{svn_urljoin(external_repo_url, 'foo.sh')} foo.sh\n" + f"{svn_urljoin(repo_url, 'trunk/bar.sh')} bar.sh" + ) + }, + ), + ], + ) + + # first load + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + # third commit + add_commit( + repo_url, + "Unset svn:externals property on trunk/externals path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={"svn:externals": None}, + ), + ], + ) + + # second load + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_with_invalid_svn_externals(swh_storage, repo_url, tmp_path): + + # first commit + add_commit( + repo_url, + "Create repository structure.", + [ + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="branches/",), + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="tags/",), + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",), + ], + ) + + # second commit + add_commit( + repo_url, + ( + "Set svn:externals property on trunk/externals path of repository to load." + "The externals URLs are not valid." + ), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + "file:///tmp/invalid/svn/repo/hello hello\n" + "file:///tmp/invalid/svn/repo/foo.sh foo.sh" + ) + }, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_with_valid_externals_modification( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create some directories and files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/hello/hello-world", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho Hello World !", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/bar/bar.sh", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho bar", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="foo.sh", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # first commit + add_commit( + repo_url, + ("Set svn:externals property on trunk/externals path of repository to load."), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/hello')} src/code/hello\n" # noqa + f"{svn_urljoin(external_repo_url, 'foo.sh')} src/foo.sh\n" + ) + }, + ), + ], + ) + + # second commit + add_commit( + repo_url, + ( + "Modify svn:externals property on trunk/externals path of repository to load." # noqa + ), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/bar')} src/code/bar\n" # noqa + f"{svn_urljoin(external_repo_url, 'foo.sh')} src/foo.sh\n" + ) + }, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_with_valid_externals_and_versioned_path( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/script.sh", + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add file with same name but different content in main repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/script.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Add externals targeting the versioned file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/script.sh')} script.sh" # noqa + ) + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Modify the versioned file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/script.sh", + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_with_invalid_externals_and_versioned_path( + swh_storage, repo_url, tmp_path +): + + # first commit + add_commit( + repo_url, + "Add file in main repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/script.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Add invalid externals targeting the versioned file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + "file:///tmp/invalid/svn/repo/code/script.sh script.sh" + ) + }, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_set_externals_then_remove_and_add_as_local( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/script.sh", + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk directory and set externals", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") + }, + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Unset externals on trunk and add remote path as local path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={"svn:externals": None}, + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/code/script.sh", + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_set_invalid_externals_then_remove(swh_storage, repo_url, tmp_path): + + # first commit + add_commit( + repo_url, + "Add trunk directory and set invalid external", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": "file:///tmp/invalid/svn/repo/code external/code" + }, + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Unset externals on trunk", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={"svn:externals": None}, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_set_externals_with_versioned_file_overlap( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/script.sh", + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add file with same name as in the external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/script.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Set external on trunk overlapping versioned file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/script.sh')} script.sh" + ) + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Unset externals on trunk", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={"svn:externals": None}, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_dump_loader_relative_externals_detection( + swh_storage, repo_url, external_repo_url, tmp_path +): + + add_commit( + external_repo_url, + "Create a file in external repository.", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project1/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + add_commit( + external_repo_url, + "Create another file in repository to load.", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project2/bar.sh", + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + external_url = f"{external_repo_url.replace('file://', '//')}/project2/bar.sh" + add_commit( + repo_url, + "Set external relative to URL scheme in repository to load", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project1/", + properties={"svn:externals": (f"{external_url} bar.sh")}, + ), + ], + ) + + loader = SvnLoaderFromRemoteDump( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1 + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + assert loader.svnrepo.has_relative_externals + + add_commit( + repo_url, + "Unset external in repository to load", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project1/", + properties={"svn:externals": None}, + ), + ], + ) + + loader = SvnLoaderFromRemoteDump( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1 + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + assert not loader.svnrepo.has_relative_externals + + +def test_loader_externals_cache(swh_storage, repo_url, external_repo_url, tmp_path): + + # first commit on external + add_commit( + external_repo_url, + "Create some directories and files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/hello/hello-world", + properties={"svn:executable": "*"}, + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Create repository structure.", + [ + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="project1/",), + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="project2/",), + ], + ) + + external_url = svn_urljoin(external_repo_url, "code/hello") + + # second commit + add_commit( + repo_url, + ( + "Set svn:externals property on trunk/externals path of repository to load." + "One external targets a remote directory and another one a remote file." + ), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project1/externals/", + properties={"svn:externals": (f"{external_url} hello\n")}, + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="project2/externals/", + properties={"svn:externals": (f"{external_url} hello\n")}, + ), + ], + ) + + loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + assert ( + external_url, + None, + False, + ) in loader.svnrepo.swhreplay.editor.externals_cache + + +def test_loader_remove_versioned_path_with_external_overlap( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/hello.sh", + data=b"#!/bin/bash\necho Hello World !", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add a file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/project/script.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Set external on trunk overlapping versioned path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code')} project/code" + ) + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Remove trunk/project/ versioned path", + [CommitChange(change_type=CommitChangeType.Delete, path="trunk/project/",),], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_export_external_path_using_peg_rev( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # second commit on external + add_commit( + external_repo_url, + "Remove previously added file", + [CommitChange(change_type=CommitChangeType.Delete, path="code/foo.sh",),], + ) + + # third commit on external + add_commit( + external_repo_url, + "Add file again but with different content", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/foo.sh", + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk dir", + [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",),], + ) + + # second commit + add_commit( + repo_url, + "Set external on trunk targeting first revision of external repo", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/foo.sh')}@1 foo.sh" + ) + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Modify external on trunk to target third revision of external repo", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/foo.sh')}@3 foo.sh" + ) + }, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_remove_external_overlapping_versioned_path( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/link", + data=b"#!/bin/bash\necho link", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk dir and a link file", + [ + CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/"), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/link", + data=b"link ../test", + properties={"svn:special": "*"}, + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Set external on root dir overlapping versioned trunk path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="", # repo root dir + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code/foo.sh')} trunk/code/foo.sh\n" # noqa + f"{svn_urljoin(external_repo_url, 'code/link')} trunk/link" + ) + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Remove external on root dir", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="", + properties={"svn:externals": None}, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_modify_external_same_path( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk dir", + [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/")], + ) + + # second commit + add_commit( + repo_url, + "Set external code on trunk dir", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") + }, + ), + ], + ) + + # third commit + add_commit( + repo_url, + "Change code external on trunk targeting an invalid URL", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={"svn:externals": "file:///tmp/invalid/svn/repo/path code"}, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_with_recursive_external( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="code/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk dir and a file", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/bar.sh", + data=b"#!/bin/bash\necho bar", + ) + ], + ) + + # second commit + add_commit( + repo_url, + "Set externals code on trunk/externals dir, one being recursive", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'code')} code\n" + f"{repo_url} recursive" + ) + }, + ), + ], + ) + + # first load + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + assert loader.svnrepo.has_recursive_externals + + # second load on stale repo + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "uneventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + assert loader.svnrepo.has_recursive_externals + + # third commit + add_commit( + repo_url, + "Remove recursive external on trunk/externals dir", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") + }, + ), + ], + ) + + # third load + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + assert not loader.svnrepo.has_recursive_externals + + +def test_loader_externals_with_same_target( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="foo/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="bar/bar.sh", + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk/src dir", + [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/src/")], + ) + + # second commit + add_commit( + repo_url, + "Add externals on trunk targeting same directory", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'foo')} src\n" + f"{svn_urljoin(external_repo_url, 'bar')} src" + ) + }, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_loader_external_in_versioned_path( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="src/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Add trunk/src dir", + [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/src/")], + ) + + # second commit + add_commit( + repo_url, + "Add a file in trunk/src directory and set external on trunk targeting src", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/src/bar.sh", + data=b"#!/bin/bash\necho bar", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": (f"{svn_urljoin(external_repo_url, 'src')} src") + }, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) + + +def test_dump_loader_externals_in_loaded_repository(swh_storage, tmp_path, mocker): + repo_url = create_repo(tmp_path, repo_name="foo") + externa_url = create_repo(tmp_path, repo_name="foobar") + + # first commit on external + add_commit( + externa_url, + "Create a file in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/src/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + ], + ) + + add_commit( + repo_url, + ( + "Add a file and set externals on trunk/externals:" + "one external located in this repository, the other in a remote one" + ), + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/src/bar.sh", + data=b"#!/bin/bash\necho bar", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/externals/", + properties={ + "svn:externals": ( + f"{svn_urljoin(repo_url, 'trunk/src/bar.sh')} bar.sh\n" + f"{svn_urljoin(externa_url, 'trunk/src/foo.sh')} foo.sh" + ) + }, + ), + ], + ) + + from swh.loader.svn.svn import client + + mock_client = mocker.MagicMock() + mocker.patch.object(client, "Client", mock_client) + + loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=tmp_path) + loader.load() + + export_call_args = mock_client().export.call_args_list + + # first external export should use the base URL of the local repository + # mounted from the remote dump as it is located in loaded repository + assert export_call_args[0][0][0] != svn_urljoin( + loader.svnrepo.origin_url, "trunk/src/bar.sh" + ) + assert export_call_args[0][0][0] == svn_urljoin( + loader.svnrepo.remote_url, "trunk/src/bar.sh" + ) + + # second external export should use the remote URL of the external repository + assert export_call_args[1][0][0] == svn_urljoin(externa_url, "trunk/src/foo.sh") + + +def test_loader_externals_add_remove_readd_on_subpath( + swh_storage, repo_url, external_repo_url, tmp_path +): + # first commit on external + add_commit( + external_repo_url, + "Create files in an external repository", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="src/foo.sh", + data=b"#!/bin/bash\necho foo", + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="src/bar.sh", + data=b"#!/bin/bash\necho bar", + ), + ], + ) + + # first commit + add_commit( + repo_url, + "Set external on two paths targeting the same absolute path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/src/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'src/foo.sh')} foo.sh" + ) + }, + ), + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'src/foo.sh')} src/foo.sh" + ) + }, + ), + ], + ) + + # second commit + add_commit( + repo_url, + "Remove external on a single path", + [ + CommitChange( + change_type=CommitChangeType.AddOrUpdate, + path="trunk/", + properties={ + "svn:externals": ( + f"{svn_urljoin(external_repo_url, 'src/bar.sh')} src/bar.sh" + ) + }, + ), + ], + ) + + loader = SvnLoader( + swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, + ) + assert loader.load() == {"status": "eventful"} + assert_last_visit_matches( + loader.storage, repo_url, status="full", type="svn", + ) + check_snapshot(loader.snapshot, loader.storage) diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -3,17 +3,13 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from enum import Enum -from io import BytesIO import os import shutil import subprocess -from typing import Any, Dict, List +from typing import Any, Dict import pytest -from subvertpy import SubversionException, delta, repos -from subvertpy.ra import Auth, RemoteAccess, get_username_provider -from typing_extensions import TypedDict +from subvertpy import SubversionException from swh.loader.svn.loader import ( SvnLoader, @@ -21,7 +17,7 @@ SvnLoaderFromRemoteDump, ) from swh.loader.svn.svn import SvnRepo -from swh.loader.svn.utils import init_svn_repo_from_dump, svn_urljoin +from swh.loader.svn.utils import init_svn_repo_from_dump from swh.loader.tests import ( assert_last_visit_matches, check_snapshot, @@ -32,6 +28,8 @@ from swh.model.hashutil import hash_to_bytes from swh.model.model import Snapshot, SnapshotBranch, TargetType +from .utils import CommitChange, CommitChangeType, add_commit + GOURMET_SNAPSHOT = Snapshot( id=hash_to_bytes("889cacc2731e3312abfb2b1a0c18ade82a949e07"), branches={ @@ -953,74 +951,6 @@ } -class CommitChangeType(Enum): - AddOrUpdate = 1 - Delete = 2 - - -class CommitChange(TypedDict, total=False): - change_type: CommitChangeType - path: str - properties: Dict[str, str] - data: bytes - - -def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None: - conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()])) - editor = conn.get_commit_editor({"svn:log": message}) - root = editor.open_root() - for change in changes: - if change["change_type"] == CommitChangeType.Delete: - root.delete_entry(change["path"].rstrip("/")) - else: - dir_change = change["path"].endswith("/") - split_path = change["path"].rstrip("/").split("/") - for i in range(len(split_path)): - path = "/".join(split_path[0 : i + 1]) - if i < len(split_path) - 1: - try: - root.add_directory(path).close() - except SubversionException: - pass - else: - if dir_change: - try: - dir = root.add_directory(path) - except SubversionException: - dir = root.open_directory(path) - if "properties" in change: - for prop, value in change["properties"].items(): - dir.change_prop(prop, value) - dir.close() - else: - try: - file = root.add_file(path) - except SubversionException: - file = root.open_file(path) - if "properties" in change: - for prop, value in change["properties"].items(): - file.change_prop(prop, value) - if "data" in change: - txdelta = file.apply_textdelta() - delta.send_stream(BytesIO(change["data"]), txdelta) - file.close() - root.close() - editor.close() - - -def create_repo(tmp_path, repo_name="tmprepo"): - repo_path = os.path.join(tmp_path, repo_name) - repos.create(repo_path) - repo_url = f"file://{repo_path}" - return repo_url - - -@pytest.fixture -def repo_url(tmpdir_factory): - # create a repository - return create_repo(tmpdir_factory.mktemp("repos")) - - def test_loader_eol_style_file_property_handling_edge_case( swh_storage, repo_url, tmp_path ): @@ -1848,1258 +1778,3 @@ loader.storage, repo_url, status="full", type="svn", ) check_snapshot(loader.snapshot, loader.storage) - - -@pytest.fixture -def external_repo_url(tmpdir_factory): - # create a repository - return create_repo(tmpdir_factory.mktemp("external")) - - -def test_loader_with_valid_svn_externals( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create some directories and files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/hello/hello-world", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho Hello World !", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="foo.sh", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Create repository structure.", - [ - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="branches/",), - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="tags/",), - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/bar.sh", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - # second commit - add_commit( - repo_url, - ( - "Set svn:externals property on trunk/externals path of repository to load." - "One external targets a remote directory and another one a remote file." - ), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/hello')} hello\n" - f"{svn_urljoin(external_repo_url, 'foo.sh')} foo.sh\n" - f"{svn_urljoin(repo_url, 'trunk/bar.sh')} bar.sh" - ) - }, - ), - ], - ) - - # first load - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - # third commit - add_commit( - repo_url, - "Unset svn:externals property on trunk/externals path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={"svn:externals": None}, - ), - ], - ) - - # second load - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_with_invalid_svn_externals(swh_storage, repo_url, tmp_path): - - # first commit - add_commit( - repo_url, - "Create repository structure.", - [ - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="branches/",), - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="tags/",), - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",), - ], - ) - - # second commit - add_commit( - repo_url, - ( - "Set svn:externals property on trunk/externals path of repository to load." - "The externals URLs are not valid." - ), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - "file:///tmp/invalid/svn/repo/hello hello\n" - "file:///tmp/invalid/svn/repo/foo.sh foo.sh" - ) - }, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_with_valid_externals_modification( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create some directories and files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/hello/hello-world", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho Hello World !", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/bar/bar.sh", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho bar", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="foo.sh", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # first commit - add_commit( - repo_url, - ("Set svn:externals property on trunk/externals path of repository to load."), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/hello')} src/code/hello\n" # noqa - f"{svn_urljoin(external_repo_url, 'foo.sh')} src/foo.sh\n" - ) - }, - ), - ], - ) - - # second commit - add_commit( - repo_url, - ( - "Modify svn:externals property on trunk/externals path of repository to load." # noqa - ), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/bar')} src/code/bar\n" # noqa - f"{svn_urljoin(external_repo_url, 'foo.sh')} src/foo.sh\n" - ) - }, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_with_valid_externals_and_versioned_path( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/script.sh", - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add file with same name but different content in main repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/script.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Add externals targeting the versioned file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/script.sh')} script.sh" # noqa - ) - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Modify the versioned file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/script.sh", - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_with_invalid_externals_and_versioned_path( - swh_storage, repo_url, tmp_path -): - - # first commit - add_commit( - repo_url, - "Add file in main repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/script.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Add invalid externals targeting the versioned file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - "file:///tmp/invalid/svn/repo/code/script.sh script.sh" - ) - }, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_set_externals_then_remove_and_add_as_local( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/script.sh", - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk directory and set externals", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") - }, - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Unset externals on trunk and add remote path as local path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={"svn:externals": None}, - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/code/script.sh", - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_set_invalid_externals_then_remove(swh_storage, repo_url, tmp_path): - - # first commit - add_commit( - repo_url, - "Add trunk directory and set invalid external", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": "file:///tmp/invalid/svn/repo/code external/code" - }, - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Unset externals on trunk", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={"svn:externals": None}, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_set_externals_with_versioned_file_overlap( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/script.sh", - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add file with same name as in the external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/script.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Set external on trunk overlapping versioned file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/script.sh')} script.sh" - ) - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Unset externals on trunk", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={"svn:externals": None}, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_dump_loader_relative_externals_detection( - swh_storage, repo_url, external_repo_url, tmp_path -): - - add_commit( - external_repo_url, - "Create a file in external repository.", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project1/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - add_commit( - external_repo_url, - "Create another file in repository to load.", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project2/bar.sh", - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - external_url = f"{external_repo_url.replace('file://', '//')}/project2/bar.sh" - add_commit( - repo_url, - "Set external relative to URL scheme in repository to load", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project1/", - properties={"svn:externals": (f"{external_url} bar.sh")}, - ), - ], - ) - - loader = SvnLoaderFromRemoteDump( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1 - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - assert loader.svnrepo.has_relative_externals - - add_commit( - repo_url, - "Unset external in repository to load", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project1/", - properties={"svn:externals": None}, - ), - ], - ) - - loader = SvnLoaderFromRemoteDump( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1 - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - assert not loader.svnrepo.has_relative_externals - - -def test_loader_externals_cache(swh_storage, repo_url, external_repo_url, tmp_path): - - # first commit on external - add_commit( - external_repo_url, - "Create some directories and files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/hello/hello-world", - properties={"svn:executable": "*"}, - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Create repository structure.", - [ - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="project1/",), - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="project2/",), - ], - ) - - external_url = svn_urljoin(external_repo_url, "code/hello") - - # second commit - add_commit( - repo_url, - ( - "Set svn:externals property on trunk/externals path of repository to load." - "One external targets a remote directory and another one a remote file." - ), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project1/externals/", - properties={"svn:externals": (f"{external_url} hello\n")}, - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="project2/externals/", - properties={"svn:externals": (f"{external_url} hello\n")}, - ), - ], - ) - - loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - assert ( - external_url, - None, - False, - ) in loader.svnrepo.swhreplay.editor.externals_cache - - -def test_loader_remove_versioned_path_with_external_overlap( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/hello.sh", - data=b"#!/bin/bash\necho Hello World !", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add a file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/project/script.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Set external on trunk overlapping versioned path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code')} project/code" - ) - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Remove trunk/project/ versioned path", - [CommitChange(change_type=CommitChangeType.Delete, path="trunk/project/",),], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_export_external_path_using_peg_rev( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # second commit on external - add_commit( - external_repo_url, - "Remove previously added file", - [CommitChange(change_type=CommitChangeType.Delete, path="code/foo.sh",),], - ) - - # third commit on external - add_commit( - external_repo_url, - "Add file again but with different content", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/foo.sh", - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk dir", - [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/",),], - ) - - # second commit - add_commit( - repo_url, - "Set external on trunk targeting first revision of external repo", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/foo.sh')}@1 foo.sh" - ) - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Modify external on trunk to target third revision of external repo", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/foo.sh')}@3 foo.sh" - ) - }, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_remove_external_overlapping_versioned_path( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/link", - data=b"#!/bin/bash\necho link", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk dir and a link file", - [ - CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/"), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/link", - data=b"link ../test", - properties={"svn:special": "*"}, - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Set external on root dir overlapping versioned trunk path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="", # repo root dir - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code/foo.sh')} trunk/code/foo.sh\n" # noqa - f"{svn_urljoin(external_repo_url, 'code/link')} trunk/link" - ) - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Remove external on root dir", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="", - properties={"svn:externals": None}, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_modify_external_same_path( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk dir", - [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/")], - ) - - # second commit - add_commit( - repo_url, - "Set external code on trunk dir", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") - }, - ), - ], - ) - - # third commit - add_commit( - repo_url, - "Change code external on trunk targeting an invalid URL", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={"svn:externals": "file:///tmp/invalid/svn/repo/path code"}, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_with_recursive_external( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="code/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk dir and a file", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/bar.sh", - data=b"#!/bin/bash\necho bar", - ) - ], - ) - - # second commit - add_commit( - repo_url, - "Set externals code on trunk/externals dir, one being recursive", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'code')} code\n" - f"{repo_url} recursive" - ) - }, - ), - ], - ) - - # first load - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - assert loader.svnrepo.has_recursive_externals - - # second load on stale repo - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "uneventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - assert loader.svnrepo.has_recursive_externals - - # third commit - add_commit( - repo_url, - "Remove recursive external on trunk/externals dir", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": (f"{svn_urljoin(external_repo_url, 'code')} code") - }, - ), - ], - ) - - # third load - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - assert not loader.svnrepo.has_recursive_externals - - -def test_loader_externals_with_same_target( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="foo/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="bar/bar.sh", - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk/src dir", - [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/src/")], - ) - - # second commit - add_commit( - repo_url, - "Add externals on trunk targeting same directory", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'foo')} src\n" - f"{svn_urljoin(external_repo_url, 'bar')} src" - ) - }, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_loader_external_in_versioned_path( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="src/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Add trunk/src dir", - [CommitChange(change_type=CommitChangeType.AddOrUpdate, path="trunk/src/")], - ) - - # second commit - add_commit( - repo_url, - "Add a file in trunk/src directory and set external on trunk targeting src", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/src/bar.sh", - data=b"#!/bin/bash\necho bar", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": (f"{svn_urljoin(external_repo_url, 'src')} src") - }, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) - - -def test_dump_loader_externals_in_loaded_repository(swh_storage, tmp_path, mocker): - repo_url = create_repo(tmp_path, repo_name="foo") - externa_url = create_repo(tmp_path, repo_name="foobar") - - # first commit on external - add_commit( - externa_url, - "Create a file in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/src/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - ], - ) - - add_commit( - repo_url, - ( - "Add a file and set externals on trunk/externals:" - "one external located in this repository, the other in a remote one" - ), - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/src/bar.sh", - data=b"#!/bin/bash\necho bar", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/externals/", - properties={ - "svn:externals": ( - f"{svn_urljoin(repo_url, 'trunk/src/bar.sh')} bar.sh\n" - f"{svn_urljoin(externa_url, 'trunk/src/foo.sh')} foo.sh" - ) - }, - ), - ], - ) - - from swh.loader.svn.svn import client - - mock_client = mocker.MagicMock() - mocker.patch.object(client, "Client", mock_client) - - loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=tmp_path) - loader.load() - - export_call_args = mock_client().export.call_args_list - - # first external export should use the base URL of the local repository - # mounted from the remote dump as it is located in loaded repository - assert export_call_args[0][0][0] != svn_urljoin( - loader.svnrepo.origin_url, "trunk/src/bar.sh" - ) - assert export_call_args[0][0][0] == svn_urljoin( - loader.svnrepo.remote_url, "trunk/src/bar.sh" - ) - - # second external export should use the remote URL of the external repository - assert export_call_args[1][0][0] == svn_urljoin(externa_url, "trunk/src/foo.sh") - - -def test_loader_externals_add_remove_readd_on_subpath( - swh_storage, repo_url, external_repo_url, tmp_path -): - # first commit on external - add_commit( - external_repo_url, - "Create files in an external repository", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="src/foo.sh", - data=b"#!/bin/bash\necho foo", - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="src/bar.sh", - data=b"#!/bin/bash\necho bar", - ), - ], - ) - - # first commit - add_commit( - repo_url, - "Set external on two paths targeting the same absolute path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/src/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'src/foo.sh')} foo.sh" - ) - }, - ), - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'src/foo.sh')} src/foo.sh" - ) - }, - ), - ], - ) - - # second commit - add_commit( - repo_url, - "Remove external on a single path", - [ - CommitChange( - change_type=CommitChangeType.AddOrUpdate, - path="trunk/", - properties={ - "svn:externals": ( - f"{svn_urljoin(external_repo_url, 'src/bar.sh')} src/bar.sh" - ) - }, - ), - ], - ) - - loader = SvnLoader( - swh_storage, repo_url, temp_directory=tmp_path, check_revision=1, - ) - assert loader.load() == {"status": "eventful"} - assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", - ) - check_snapshot(loader.snapshot, loader.storage) diff --git a/swh/loader/svn/tests/utils.py b/swh/loader/svn/tests/utils.py new file mode 100644 --- /dev/null +++ b/swh/loader/svn/tests/utils.py @@ -0,0 +1,74 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from enum import Enum +from io import BytesIO +import os +from typing import Dict, List + +from subvertpy import SubversionException, delta, repos +from subvertpy.ra import Auth, RemoteAccess, get_username_provider +from typing_extensions import TypedDict + + +class CommitChangeType(Enum): + AddOrUpdate = 1 + Delete = 2 + + +class CommitChange(TypedDict, total=False): + change_type: CommitChangeType + path: str + properties: Dict[str, str] + data: bytes + + +def add_commit(repo_url: str, message: str, changes: List[CommitChange]) -> None: + conn = RemoteAccess(repo_url, auth=Auth([get_username_provider()])) + editor = conn.get_commit_editor({"svn:log": message}) + root = editor.open_root() + for change in changes: + if change["change_type"] == CommitChangeType.Delete: + root.delete_entry(change["path"].rstrip("/")) + else: + dir_change = change["path"].endswith("/") + split_path = change["path"].rstrip("/").split("/") + for i in range(len(split_path)): + path = "/".join(split_path[0 : i + 1]) + if i < len(split_path) - 1: + try: + root.add_directory(path).close() + except SubversionException: + pass + else: + if dir_change: + try: + dir = root.add_directory(path) + except SubversionException: + dir = root.open_directory(path) + if "properties" in change: + for prop, value in change["properties"].items(): + dir.change_prop(prop, value) + dir.close() + else: + try: + file = root.add_file(path) + except SubversionException: + file = root.open_file(path) + if "properties" in change: + for prop, value in change["properties"].items(): + file.change_prop(prop, value) + if "data" in change: + txdelta = file.apply_textdelta() + delta.send_stream(BytesIO(change["data"]), txdelta) + file.close() + root.close() + editor.close() + + +def create_repo(tmp_path, repo_name="tmprepo"): + repo_path = os.path.join(tmp_path, repo_name) + repos.create(repo_path) + return f"file://{repo_path}"