diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -659,6 +659,9 @@ If the svnrdump command failed somehow, the produced dump file is analyzed to determine if a partial loading is still feasible. + + Raises: + NotFound when the repository is no longer found at url """ # Build the svnrdump command line svnrdump_cmd = ["svnrdump", "dump", svn_url] @@ -741,6 +744,13 @@ % (last_dumped_rev, last_loaded_svn_rev) ) + if svnrdump.returncode == 1: + # Try to detect not found repositories + for stderr_line in stderr_lines: + if "E170013:" in stderr_line: + # No longer a repository at this location + raise NotFound + raise Exception( "An error occurred when running svnrdump and " "no exploitable dump file has been generated." diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -92,19 +92,40 @@ ) +def test_loader_svnrdump_not_found(swh_storage, tmp_path, mocker): + """Given any errors raised, the loader visit ends up in status failed""" + unknown_repo_url = "file:///tmp/svn.code.sf.net/p/white-rats-studios/svn" + + # mock = mocker.patch("swh.loader.svn.loader.SvnRepo") + # mock.side_effect = exception + + # existing_repo_url = "existing-repo-url" + loader = SvnLoaderFromRemoteDump( + swh_storage, unknown_repo_url, destination_path=tmp_path + ) + + assert loader.load() == {"status": "uneventful"} + + assert_last_visit_matches( + swh_storage, unknown_repo_url, status="not_found", type="svn", + ) + + def test_loader_svn_new_visit(swh_storage, datadir, tmp_path): """Eventful visit should yield 1 snapshot""" archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) + loader = SvnLoader(swh_storage, unknown_repo_url, destination_path=tmp_path) assert loader.load() == {"status": "eventful"} assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, @@ -131,14 +152,16 @@ """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} visit_status1 = assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, @@ -147,7 +170,7 @@ assert loader.load() == {"status": "uneventful"} visit_status2 = assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, @@ -166,7 +189,7 @@ )[0] assert start_revision is not None - loader = SvnLoader(swh_storage, repo_url, swh_revision=start_revision) + loader = SvnLoader(swh_storage, unknown_repo_url, swh_revision=start_revision) assert loader.load() == {"status": "uneventful"} stats = get_stats(loader.storage) @@ -176,7 +199,7 @@ assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, @@ -202,9 +225,11 @@ """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} check_snapshot(GOURMET_SNAPSHOT, loader.storage) @@ -213,11 +238,11 @@ archive_path2, archive_name, tmp_path ) - loader2 = SvnLoader(swh_storage, repo_tampered_url, origin_url=repo_url) + loader2 = SvnLoader(swh_storage, repo_tampered_url, origin_url=unknown_repo_url) assert loader2.load() == {"status": "failed"} assert_last_visit_matches( - loader2.storage, repo_url, status="failed", type="svn", snapshot=None, + loader2.storage, unknown_repo_url, status="failed", type="svn", snapshot=None, ) stats = get_stats(loader.storage) @@ -388,9 +413,11 @@ """ archive_name = "mediawiki-repo-r407-eol-native-crlf" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} mediawiki_snapshot = Snapshot( @@ -406,7 +433,7 @@ assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=mediawiki_snapshot.id, @@ -428,9 +455,11 @@ """ archive_name = "pyang-repo-r343-eol-native-mixed-lf-crlf" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} pyang_snapshot = Snapshot( @@ -445,7 +474,11 @@ check_snapshot(pyang_snapshot, loader.storage) assert_last_visit_matches( - loader.storage, repo_url, status="full", type="svn", snapshot=pyang_snapshot.id, + loader.storage, + unknown_repo_url, + status="full", + type="svn", + snapshot=pyang_snapshot.id, ) stats = get_stats(loader.storage) @@ -460,9 +493,11 @@ """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, "pkg-gourmet-with-external-id.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} gourmet_externals_snapshot = Snapshot( @@ -477,7 +512,7 @@ check_snapshot(gourmet_externals_snapshot, loader.storage) assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="partial", type="svn", snapshot=gourmet_externals_snapshot.id, @@ -506,9 +541,11 @@ archive_path = os.path.join( datadir, "pkg-gourmet-with-edge-case-links-and-files.tgz" ) - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} gourmet_edge_cases_snapshot = Snapshot( @@ -524,7 +561,7 @@ assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=gourmet_edge_cases_snapshot.id, @@ -547,9 +584,11 @@ """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, "pkg-gourmet-with-wrong-link-cases.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} gourmet_wrong_links_snapshot = Snapshot( @@ -565,7 +604,7 @@ assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=gourmet_wrong_links_snapshot.id, @@ -588,20 +627,22 @@ """ archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loaderFromDump = SvnLoaderFromRemoteDump(swh_storage, repo_url) + loaderFromDump = SvnLoaderFromRemoteDump(swh_storage, unknown_repo_url) assert loaderFromDump.load() == {"status": "eventful"} assert_last_visit_matches( loaderFromDump.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=GOURMET_SNAPSHOT.id, ) - origin_url = repo_url + "2" # rename to another origin - loader = SvnLoader(swh_storage, repo_url, origin_url=origin_url) + origin_url = unknown_repo_url + "2" # rename to another origin + loader = SvnLoader(swh_storage, unknown_repo_url, origin_url=origin_url) assert loader.load() == {"status": "eventful"} # because are working on new origin assert_last_visit_matches( loader.storage, @@ -618,7 +659,7 @@ assert stats["origin_visit"] == 2 assert stats["snapshot"] == 1 - loader = SvnLoader(swh_storage, repo_url) # no change on the origin-url + loader = SvnLoader(swh_storage, unknown_repo_url) # no change on the origin-url assert loader.load() == {"status": "uneventful"} assert_last_visit_matches( loader.storage, @@ -634,7 +675,7 @@ assert stats["snapshot"] == 1 # second visit from the dump should be uneventful - loaderFromDump = SvnLoaderFromRemoteDump(swh_storage, repo_url) + loaderFromDump = SvnLoaderFromRemoteDump(swh_storage, unknown_repo_url) assert loaderFromDump.load() == {"status": "uneventful"} @@ -646,9 +687,11 @@ """ archive_name = "httthttt" archive_path = os.path.join(datadir, f"{archive_name}.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url) + loader = SvnLoader(swh_storage, unknown_repo_url) assert loader.load() == {"status": "eventful"} expected_snapshot = Snapshot( @@ -664,7 +707,7 @@ assert_last_visit_matches( loader.storage, - repo_url, + unknown_repo_url, status="full", type="svn", snapshot=expected_snapshot.id, @@ -681,9 +724,11 @@ """Loader should handle directory removal when processing a commit""" archive_name = "pkg-gourmet" archive_path = os.path.join(datadir, f"{archive_name}-add-remove-dir.tgz") - repo_url = prepare_repository_from_archive(archive_path, archive_name, tmp_path) + unknown_repo_url = prepare_repository_from_archive( + archive_path, archive_name, tmp_path + ) - loader = SvnLoader(swh_storage, repo_url, destination_path=tmp_path) + loader = SvnLoader(swh_storage, unknown_repo_url, destination_path=tmp_path) assert loader.load() == {"status": "eventful"} assert loader.visit_status() == "full"