diff --git a/mypy.ini b/mypy.ini --- a/mypy.ini +++ b/mypy.ini @@ -14,5 +14,5 @@ [mypy-pytest.*] ignore_missing_imports = True -[mypy-swh.loader.package.*] +[mypy-swh.loader.*] ignore_missing_imports = True diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,4 +1,4 @@ -swh.storage >= 0.1 +swh.storage >= 0.5.0 swh.model >= 0.3 swh.scheduler >= 0.0.39 swh.loader.core >= 0.2 diff --git a/swh/loader/svn/loader.py b/swh/loader/svn/loader.py --- a/swh/loader/svn/loader.py +++ b/swh/loader/svn/loader.py @@ -124,6 +124,7 @@ self.swh_revision = swh_revision self.max_content_length = self.config["max_content_size"] self.snapshot = None + self.last_visit = None def pre_cleanup(self): """Cleanup potential dangling files from prior runs (e.g. OOM killed @@ -166,13 +167,14 @@ self.svnrepo.clean_fs(local_dirname) return h - def swh_latest_snapshot_revision(self, origin_url, previous_swh_revision=None): + def swh_latest_snapshot_revision( + self, origin_url: str, previous_swh_revision: Optional[bytes] = None + ): """Look for latest snapshot revision and returns it if any. Args: - origin_url (str): Origin identifier - previous_swh_revision: (optional) id of a possible - previous swh revision + origin_url: Origin identifier + previous_swh_revision: id of a possible previous swh revision Returns: dict: The latest known point in time. Dict with keys: @@ -180,14 +182,21 @@ 'revision': latest visited revision 'snapshot': latest snapshot - If None is found, return an empty dict. + If nothing matching criteria is found, return an empty dict. """ storage = self.storage + if not previous_swh_revision: # check latest snapshot's revision - visit = storage.origin_visit_get_latest(origin_url, require_snapshot=True) - if visit: - latest_snap = snapshot_get_all_branches(storage, visit["snapshot"]) + if self.last_visit is None: + return {} + visit_id = self.last_visit.visit + assert visit_id is not None + visit_status = storage.origin_visit_status_get_latest( + origin_url, visit_id, require_snapshot=True + ) + if visit_status: + latest_snap = snapshot_get_all_branches(storage, visit_status.snapshot) if latest_snap: branches = latest_snap.get("branches") if not branches: @@ -452,6 +461,7 @@ def prepare_origin_visit(self, *args, **kwargs): self.origin = Origin(url=self.origin_url if self.origin_url else self.svn_url) + self.last_visit = self.storage.origin_visit_get_latest(self.origin_url) def prepare(self, *args, **kwargs): if self.swh_revision: diff --git a/swh/loader/svn/tests/test_loader.py b/swh/loader/svn/tests/test_loader.py --- a/swh/loader/svn/tests/test_loader.py +++ b/swh/loader/svn/tests/test_loader.py @@ -7,6 +7,8 @@ import os from swh.loader.core.tests import BaseLoaderTest +from swh.loader.tests.common import assert_last_visit_matches + from swh.loader.svn.loader import ( DEFAULT_BRANCH, SvnLoader, @@ -205,9 +207,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_SNAPSHOT, + ) _LAST_SNP_REV = { @@ -253,9 +259,14 @@ self.assertCountReleases(0) self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_FLAG_SNAPSHOT) - self.assertEqual(visit["status"], "full") + + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_FLAG_SNAPSHOT, + ) class SvnLoaderTest3(BaseSvnLoaderTest): @@ -296,9 +307,10 @@ self.assertCountSnapshots(0) self.assertEqual(self.loader.visit_status(), "partial") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], None) - self.assertEqual(visit["status"], "partial") + visit_status = assert_last_visit_matches( + self.storage, self.repo_url, status="partial", type="svn" + ) + assert visit_status.snapshot is None class SvnLoaderTest4(BaseSvnLoaderTest): @@ -344,9 +356,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_UPDATES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_UPDATES_SNAPSHOT, + ) class SvnLoaderTest5(BaseSvnLoaderTest): @@ -398,9 +414,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_UPDATES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_UPDATES_SNAPSHOT, + ) class SvnLoaderTest6(BaseSvnLoaderTest): @@ -469,9 +489,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_UPDATES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_UPDATES_SNAPSHOT, + ) class SvnLoaderTest7(BaseSvnLoaderTest): @@ -535,9 +559,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_UPDATES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_UPDATES_SNAPSHOT, + ) class SvnLoaderTest8(BaseSvnLoaderTest): @@ -622,9 +650,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_UPDATES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_UPDATES_SNAPSHOT, + ) class SvnLoaderTest9(BaseSvnLoaderTest): @@ -655,9 +687,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], MEDIAWIKI_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=MEDIAWIKI_SNAPSHOT, + ) class SvnLoaderTest10(BaseSvnLoaderTest): # noqa @@ -689,9 +725,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], PYANG_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=PYANG_SNAPSHOT, + ) class SvnLoaderTest11(BaseSvnLoaderTest): @@ -753,9 +793,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "partial") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_EXTERNALS_SNAPSHOT) - self.assertEqual(visit["status"], "partial") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="partial", + type="svn", + snapshot=GOURMET_EXTERNALS_SNAPSHOT, + ) class SvnLoaderTest12(BaseSvnLoaderTest): @@ -813,9 +857,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_EDGE_CASES_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_EDGE_CASES_SNAPSHOT, + ) class SvnLoaderTest13(BaseSvnLoaderTest): @@ -870,9 +918,13 @@ self.assertCountSnapshots(1) self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_WRONG_LINKS_SNAPSHOT) - self.assertEqual(visit["status"], "full") + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_WRONG_LINKS_SNAPSHOT, + ) class SvnLoaderTestFromRemoteDump(SvnLoaderTest, SvnLoaderFromRemoteDump): @@ -908,11 +960,21 @@ base_storage_stat = base_loader.storage.stat_counters() self.assertEqual(dump_storage_stat, base_storage_stat) - visit = dump_loader.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_SNAPSHOT) + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_SNAPSHOT, + ) - visit = base_loader.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual(visit["snapshot"], GOURMET_SNAPSHOT) + assert_last_visit_matches( + base_loader.storage, + self.repo_url, + status="full", + type="svn", + snapshot=GOURMET_SNAPSHOT, + ) class SvnLoaderTest14(BaseSvnLoaderTest): @@ -958,8 +1020,10 @@ self.assertEqual(self.loader.visit_status(), "full") - visit = self.storage.origin_visit_get_latest(self.repo_url) - self.assertEqual( - visit["snapshot"], hashutil.hash_to_bytes(expected_snapshot_id) + assert_last_visit_matches( + self.storage, + self.repo_url, + status="full", + type="svn", + snapshot=hashutil.hash_to_bytes(expected_snapshot_id), ) - self.assertEqual(visit["status"], "full")