Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/svn/tests/test_loader.py
| Show First 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | branches={ | ||||
| ) | ) | ||||
| }, | }, | ||||
| ) | ) | ||||
| def test_loader_svn_not_found_no_mock(swh_storage, tmp_path): | def test_loader_svn_not_found_no_mock(swh_storage, tmp_path): | ||||
| """Given an unknown repository, the loader visit ends up in status not_found""" | """Given an unknown repository, the loader visit ends up in status not_found""" | ||||
| repo_url = "unknown-repository" | repo_url = "unknown-repository" | ||||
| loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) | ||||
| assert loader.load() == {"status": "uneventful"} | assert loader.load() == {"status": "uneventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, repo_url, status="not_found", type="svn", | swh_storage, repo_url, status="not_found", type="svn", | ||||
| ) | ) | ||||
| @pytest.mark.parametrize( | @pytest.mark.parametrize( | ||||
| "exception_msg", ["Unable to connect to a repository at URL", "Unknown URL type",] | "exception_msg", ["Unable to connect to a repository at URL", "Unknown URL type",] | ||||
| ) | ) | ||||
| def test_loader_svn_not_found(swh_storage, tmp_path, exception_msg, mocker): | def test_loader_svn_not_found(swh_storage, tmp_path, exception_msg, mocker): | ||||
| """Given unknown repository issues, the loader visit ends up in status not_found""" | """Given unknown repository issues, the loader visit ends up in status not_found""" | ||||
| mock = mocker.patch("swh.loader.svn.loader.SvnRepo") | mock = mocker.patch("swh.loader.svn.loader.SvnRepo") | ||||
| mock.side_effect = SubversionException(exception_msg, 0) | mock.side_effect = SubversionException(exception_msg, 0) | ||||
| unknown_repo_url = "unknown-repository" | unknown_repo_url = "unknown-repository" | ||||
| loader = SvnLoader(swh_storage, unknown_repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, unknown_repo_url, temp_directory=tmp_path) | ||||
| assert loader.load() == {"status": "uneventful"} | assert loader.load() == {"status": "uneventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, unknown_repo_url, status="not_found", type="svn", | swh_storage, unknown_repo_url, status="not_found", type="svn", | ||||
| ) | ) | ||||
| @pytest.mark.parametrize( | @pytest.mark.parametrize( | ||||
| "exception", | "exception", | ||||
| [ | [ | ||||
| SubversionException("Irrelevant message, considered a failure", 10), | SubversionException("Irrelevant message, considered a failure", 10), | ||||
| SubversionException("Present but fails to read, considered a failure", 20), | SubversionException("Present but fails to read, considered a failure", 20), | ||||
| ValueError("considered a failure"), | ValueError("considered a failure"), | ||||
| ], | ], | ||||
| ) | ) | ||||
| def test_loader_svn_failures(swh_storage, tmp_path, exception, mocker): | def test_loader_svn_failures(swh_storage, tmp_path, exception, mocker): | ||||
| """Given any errors raised, the loader visit ends up in status failed""" | """Given any errors raised, the loader visit ends up in status failed""" | ||||
| mock = mocker.patch("swh.loader.svn.loader.SvnRepo") | mock = mocker.patch("swh.loader.svn.loader.SvnRepo") | ||||
| mock.side_effect = exception | mock.side_effect = exception | ||||
| existing_repo_url = "existing-repo-url" | existing_repo_url = "existing-repo-url" | ||||
| loader = SvnLoader(swh_storage, existing_repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, existing_repo_url, temp_directory=tmp_path) | ||||
| assert loader.load() == {"status": "failed"} | assert loader.load() == {"status": "failed"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, existing_repo_url, status="failed", type="svn", | swh_storage, existing_repo_url, status="failed", type="svn", | ||||
| ) | ) | ||||
| def test_loader_svnrdump_not_found(swh_storage, tmp_path, mocker): | def test_loader_svnrdump_not_found(swh_storage, tmp_path, mocker): | ||||
| """Loading from remote dump which does not exist should end up as not_found visit""" | """Loading from remote dump which does not exist should end up as not_found visit""" | ||||
| unknown_repo_url = "file:///tmp/svn.code.sf.net/p/white-rats-studios/svn" | unknown_repo_url = "file:///tmp/svn.code.sf.net/p/white-rats-studios/svn" | ||||
| loader = SvnLoaderFromRemoteDump( | loader = SvnLoaderFromRemoteDump( | ||||
| swh_storage, unknown_repo_url, destination_path=tmp_path | swh_storage, unknown_repo_url, temp_directory=tmp_path | ||||
| ) | ) | ||||
| assert loader.load() == {"status": "uneventful"} | assert loader.load() == {"status": "uneventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, unknown_repo_url, status="not_found", type="svn", | swh_storage, unknown_repo_url, status="not_found", type="svn", | ||||
| ) | ) | ||||
| Show All 16 Lines | def test_loader_svnrdump_no_such_revision(swh_storage, tmp_path, datadir): | ||||
| os.mkdir(loading_path) | os.mkdir(loading_path) | ||||
| # Prepare the dump as a local svn repository for test purposes | # Prepare the dump as a local svn repository for test purposes | ||||
| temp_dir, repo_path = init_svn_repo_from_dump( | temp_dir, repo_path = init_svn_repo_from_dump( | ||||
| archive_dump, root_dir=tmp_path, gzip=True | archive_dump, root_dir=tmp_path, gzip=True | ||||
| ) | ) | ||||
| repo_url = f"file://{repo_path}" | repo_url = f"file://{repo_path}" | ||||
| loader = SvnLoaderFromRemoteDump( | loader = SvnLoaderFromRemoteDump(swh_storage, repo_url, temp_directory=loading_path) | ||||
| swh_storage, repo_url, destination_path=loading_path | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| actual_visit = assert_last_visit_matches( | actual_visit = assert_last_visit_matches( | ||||
| swh_storage, repo_url, status="full", type="svn", | swh_storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| loader2 = SvnLoaderFromRemoteDump( | loader2 = SvnLoaderFromRemoteDump( | ||||
| swh_storage, repo_url, destination_path=loading_path | swh_storage, repo_url, temp_directory=loading_path | ||||
| ) | ) | ||||
| # Visiting a second time the same repository should be uneventful... | # Visiting a second time the same repository should be uneventful... | ||||
| assert loader2.load() == {"status": "uneventful"} | assert loader2.load() == {"status": "uneventful"} | ||||
| actual_visit2 = assert_last_visit_matches( | actual_visit2 = assert_last_visit_matches( | ||||
| swh_storage, repo_url, status="full", type="svn", | swh_storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| assert actual_visit.snapshot is not None | assert actual_visit.snapshot is not None | ||||
| # ... with the same snapshot as the first visit | # ... with the same snapshot as the first visit | ||||
| assert actual_visit2.snapshot == actual_visit.snapshot | assert actual_visit2.snapshot == actual_visit.snapshot | ||||
| def test_loader_svn_new_visit(swh_storage, datadir, tmp_path): | def test_loader_svn_new_visit(swh_storage, datadir, tmp_path): | ||||
| """Eventful visit should yield 1 snapshot""" | """Eventful visit should yield 1 snapshot""" | ||||
| archive_name = "pkg-gourmet" | archive_name = "pkg-gourmet" | ||||
| archive_path = os.path.join(datadir, f"{archive_name}.tgz") | archive_path = os.path.join(datadir, f"{archive_name}.tgz") | ||||
| repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | ||||
| loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) | ||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, | loader.storage, | ||||
| repo_url, | repo_url, | ||||
| status="full", | status="full", | ||||
| type="svn", | type="svn", | ||||
| ▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | def test_loader_svn_2_visits_no_change(swh_storage, datadir, tmp_path): | ||||
| assert stats["snapshot"] == 1 | assert stats["snapshot"] == 1 | ||||
| # even starting from previous revision... | # even starting from previous revision... | ||||
| start_revision = loader.storage.revision_get( | start_revision = loader.storage.revision_get( | ||||
| [hash_to_bytes("95edacc8848369d6fb1608e887d6d2474fd5224f")] | [hash_to_bytes("95edacc8848369d6fb1608e887d6d2474fd5224f")] | ||||
| )[0] | )[0] | ||||
| assert start_revision is not None | assert start_revision is not None | ||||
| loader = SvnLoader(swh_storage, repo_url, swh_revision=start_revision) | loader = SvnLoader(swh_storage, repo_url) | ||||
| assert loader.load() == {"status": "uneventful"} | assert loader.load() == {"status": "uneventful"} | ||||
| stats = get_stats(loader.storage) | stats = get_stats(loader.storage) | ||||
| assert stats["origin_visit"] == 2 + 1 | assert stats["origin_visit"] == 2 + 1 | ||||
| # ... with no change in repository, this yields the same snapshot | # ... with no change in repository, this yields the same snapshot | ||||
| assert stats["snapshot"] == 1 | assert stats["snapshot"] == 1 | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| ▲ Show 20 Lines • Show All 168 Lines • ▼ Show 20 Lines | def test_loader_svn_visit_start_from_revision(swh_storage, datadir, tmp_path): | ||||
| assert start_revision is not None | assert start_revision is not None | ||||
| archive_path = os.path.join(datadir, "pkg-gourmet-with-updates.tgz") | archive_path = os.path.join(datadir, "pkg-gourmet-with-updates.tgz") | ||||
| repo_updated_url = prepare_repository_from_archive( | repo_updated_url = prepare_repository_from_archive( | ||||
| archive_path, "pkg-gourmet", tmp_path | archive_path, "pkg-gourmet", tmp_path | ||||
| ) | ) | ||||
| # we'll start from start_revision | # we'll start from start_revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_updated_url, origin_url=repo_initial_url,) | ||||
| swh_storage, | |||||
| repo_updated_url, | |||||
| origin_url=repo_initial_url, | |||||
| swh_revision=start_revision, | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| # nonetheless, we obtain the same snapshot (as previous tests on that repository) | # nonetheless, we obtain the same snapshot (as previous tests on that repository) | ||||
| visit_status2 = assert_last_visit_matches( | visit_status2 = assert_last_visit_matches( | ||||
| loader.storage, | loader.storage, | ||||
| repo_updated_url, | repo_updated_url, | ||||
| status="full", | status="full", | ||||
| ▲ Show 20 Lines • Show All 318 Lines • ▼ Show 20 Lines | |||||
| def test_loader_svn_dir_added_then_removed(swh_storage, datadir, tmp_path): | def test_loader_svn_dir_added_then_removed(swh_storage, datadir, tmp_path): | ||||
| """Loader should handle directory removal when processing a commit""" | """Loader should handle directory removal when processing a commit""" | ||||
| archive_name = "pkg-gourmet" | archive_name = "pkg-gourmet" | ||||
| archive_path = os.path.join(datadir, f"{archive_name}-add-remove-dir.tgz") | archive_path = os.path.join(datadir, f"{archive_name}-add-remove-dir.tgz") | ||||
| repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | ||||
| loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) | ||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| def test_loader_svn_loader_from_dump_archive(swh_storage, datadir, tmp_path): | def test_loader_svn_loader_from_dump_archive(swh_storage, datadir, tmp_path): | ||||
| archive_name = "pkg-gourmet" | archive_name = "pkg-gourmet" | ||||
| archive_path = os.path.join(datadir, f"{archive_name}.tgz") | archive_path = os.path.join(datadir, f"{archive_name}.tgz") | ||||
| repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | ||||
| origin_url = f"svn://{archive_name}" | origin_url = f"svn://{archive_name}" | ||||
| dump_filename = f"{archive_name}.dump" | dump_filename = f"{archive_name}.dump" | ||||
| with open(os.path.join(tmp_path, dump_filename), "wb") as dump_file: | with open(os.path.join(tmp_path, dump_filename), "wb") as dump_file: | ||||
| # create compressed dump file of pkg-gourmet repo | # create compressed dump file of pkg-gourmet repo | ||||
| subprocess.run(["svnrdump", "dump", repo_url], stdout=dump_file) | subprocess.run(["svnrdump", "dump", repo_url], stdout=dump_file) | ||||
| subprocess.run(["gzip", dump_filename], cwd=tmp_path) | subprocess.run(["gzip", dump_filename], cwd=tmp_path) | ||||
| # load svn repo from that compressed dump file | # load svn repo from that compressed dump file | ||||
| loader = SvnLoaderFromDumpArchive( | loader = SvnLoaderFromDumpArchive( | ||||
| swh_storage, | swh_storage, | ||||
| url=origin_url, | url=origin_url, | ||||
| archive_path=os.path.join(tmp_path, f"{dump_filename}.gz"), | archive_path=os.path.join(tmp_path, f"{dump_filename}.gz"), | ||||
| temp_directory=tmp_path, | |||||
| ) | ) | ||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, | loader.storage, | ||||
| origin_url, | origin_url, | ||||
| status="full", | status="full", | ||||
| ▲ Show 20 Lines • Show All 109 Lines • ▼ Show 20 Lines | add_commit( | ||||
| data=b"Hello world!\r\n", | data=b"Hello world!\r\n", | ||||
| ) | ) | ||||
| ], | ], | ||||
| ) | ) | ||||
| # instantiate a svn loader checking after each processed revision that | # instantiate a svn loader checking after each processed revision that | ||||
| # the repository filesystem it reconstructed does not differ from a subversion | # the repository filesystem it reconstructed does not differ from a subversion | ||||
| # export of that revision | # export of that revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) | ||||
| swh_storage, repo_url, destination_path=tmp_path, check_revision=1 | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| assert get_stats(loader.storage) == { | assert get_stats(loader.storage) == { | ||||
| "content": 2, | "content": 2, | ||||
| ▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | add_commit( | ||||
| data=b"link ../file_with_crlf_eol.txt", | data=b"link ../file_with_crlf_eol.txt", | ||||
| ), | ), | ||||
| ], | ], | ||||
| ) | ) | ||||
| # instantiate a svn loader checking after each processed revision that | # instantiate a svn loader checking after each processed revision that | ||||
| # the repository filesystem it reconstructed does not differ from a subversion | # the repository filesystem it reconstructed does not differ from a subversion | ||||
| # export of that revision | # export of that revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) | ||||
| swh_storage, repo_url, destination_path=tmp_path, check_revision=1 | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| # check loaded objects are those expected | # check loaded objects are those expected | ||||
| assert get_stats(loader.storage) == { | assert get_stats(loader.storage) == { | ||||
| ▲ Show 20 Lines • Show All 73 Lines • ▼ Show 20 Lines | add_commit( | ||||
| properties={"svn:special": None}, | properties={"svn:special": None}, | ||||
| ), | ), | ||||
| ], | ], | ||||
| ) | ) | ||||
| # instantiate a svn loader checking after each processed revision that | # instantiate a svn loader checking after each processed revision that | ||||
| # the repository filesystem it reconstructed does not differ from a subversion | # the repository filesystem it reconstructed does not differ from a subversion | ||||
| # export of that revision | # export of that revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) | ||||
| swh_storage, repo_url, destination_path=tmp_path, check_revision=1 | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| # check loaded objects are those expected | # check loaded objects are those expected | ||||
| assert get_stats(loader.storage) == { | assert get_stats(loader.storage) == { | ||||
| ▲ Show 20 Lines • Show All 46 Lines • ▼ Show 20 Lines | add_commit( | ||||
| data=file_content, | data=file_content, | ||||
| ) | ) | ||||
| ], | ], | ||||
| ) | ) | ||||
| # instantiate a svn loader checking after each processed revision that | # instantiate a svn loader checking after each processed revision that | ||||
| # the repository filesystem it reconstructed does not differ from a subversion | # the repository filesystem it reconstructed does not differ from a subversion | ||||
| # export of that revision | # export of that revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) | ||||
| swh_storage, repo_url, destination_path=tmp_path, check_revision=1 | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| paths = get_head_revision_paths_info(loader) | paths = get_head_revision_paths_info(loader) | ||||
| # end of lines should not have been processed | # end of lines should not have been processed | ||||
| Show All 25 Lines | for filename in ("foo", "bar", "baz"): | ||||
| CommitChange( | CommitChange( | ||||
| change_type=CommitChangeType.AddOrUpdate, | change_type=CommitChangeType.AddOrUpdate, | ||||
| path=filename, | path=filename, | ||||
| data=f"{filename}\n".encode(), | data=f"{filename}\n".encode(), | ||||
| ) | ) | ||||
| ], | ], | ||||
| ) | ) | ||||
| loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path) | ||||
| # post loading will detect an issue and make a partial visit with a snapshot | # post loading will detect an issue and make a partial visit with a snapshot | ||||
| assert loader.load() == {"status": "failed"} | assert loader.load() == {"status": "failed"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="partial", type="svn", | loader.storage, repo_url, status="partial", type="svn", | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | add_commit( | ||||
| properties={"svn:special": None}, | properties={"svn:special": None}, | ||||
| ), | ), | ||||
| ], | ], | ||||
| ) | ) | ||||
| # instantiate a svn loader checking after each processed revision that | # instantiate a svn loader checking after each processed revision that | ||||
| # the repository filesystem it reconstructed does not differ from a subversion | # the repository filesystem it reconstructed does not differ from a subversion | ||||
| # export of that revision | # export of that revision | ||||
| loader = SvnLoader( | loader = SvnLoader(swh_storage, repo_url, temp_directory=tmp_path, check_revision=1) | ||||
| swh_storage, repo_url, destination_path=tmp_path, check_revision=1 | |||||
| ) | |||||
| assert loader.load() == {"status": "eventful"} | assert loader.load() == {"status": "eventful"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, repo_url, status="full", type="svn", | loader.storage, repo_url, status="full", type="svn", | ||||
| ) | ) | ||||
| def test_loader_last_revision_divergence(swh_storage, datadir, tmp_path): | def test_loader_last_revision_divergence(swh_storage, datadir, tmp_path): | ||||
| archive_name = "pkg-gourmet" | archive_name = "pkg-gourmet" | ||||
| archive_path = os.path.join(datadir, f"{archive_name}.tgz") | archive_path = os.path.join(datadir, f"{archive_name}.tgz") | ||||
| repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) | ||||
| class SvnLoaderRevisionDivergence(SvnLoader): | class SvnLoaderRevisionDivergence(SvnLoader): | ||||
| def _check_revision_divergence(self, count, rev, dir_id): | def _check_revision_divergence(self, count, rev, dir_id): | ||||
| raise ValueError("revision divergence detected") | raise ValueError("revision divergence detected") | ||||
| loader = SvnLoaderRevisionDivergence( | loader = SvnLoaderRevisionDivergence(swh_storage, repo_url, temp_directory=tmp_path) | ||||
| swh_storage, repo_url, destination_path=tmp_path | |||||
| ) | |||||
| assert loader.load() == {"status": "failed"} | assert loader.load() == {"status": "failed"} | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| loader.storage, | loader.storage, | ||||
| repo_url, | repo_url, | ||||
| status="partial", | status="partial", | ||||
| type="svn", | type="svn", | ||||
| snapshot=GOURMET_SNAPSHOT.id, | snapshot=GOURMET_SNAPSHOT.id, | ||||
| ) | ) | ||||