diff --git a/requirements-swh.txt b/requirements-swh.txt
--- a/requirements-swh.txt
+++ b/requirements-swh.txt
@@ -1,4 +1,4 @@
swh.core >= 0.0.75
swh.model >= 0.3.0
swh.scheduler
-swh.storage >= 0.3.0
+swh.storage >= 0.6.0
diff --git a/swh/loader/core/tests/test_loader.py b/swh/loader/core/tests/test_loader.py
--- a/swh/loader/core/tests/test_loader.py
+++ b/swh/loader/core/tests/test_loader.py
@@ -10,6 +10,8 @@
from swh.model.model import Origin, OriginVisit, Snapshot
from swh.loader.core.loader import BaseLoader, DVCSLoader
+from swh.loader.tests.conftest import assert_last_visit_ok
+
ORIGIN = Origin(url="some-url")
@@ -172,9 +174,8 @@
assert loader.loaded_snapshot_id is None
# And confirm that the visit doesn't reference a snapshot
- visit = loader.storage.origin_visit_get_latest(ORIGIN.url)
- assert visit["status"] == "partial"
- assert visit["snapshot"] is None
+ visit = assert_last_visit_ok(loader.storage, ORIGIN.url, status="partial")
+ assert visit.snapshot is None
class DummyDVCSLoaderExc(DummyDVCSLoader):
diff --git a/swh/loader/package/archive/tests/test_archive.py b/swh/loader/package/archive/tests/test_archive.py
--- a/swh/loader/package/archive/tests/test_archive.py
+++ b/swh/loader/package/archive/tests/test_archive.py
@@ -11,6 +11,7 @@
check_metadata_paths,
get_stats,
)
+from swh.loader.tests.conftest import assert_last_visit_ok
URL = "https://ftp.gnu.org/gnu/8sync/"
@@ -105,6 +106,7 @@
actual_load_status = loader.load()
assert actual_load_status["status"] == "uneventful"
assert actual_load_status["snapshot_id"] is not None
+
stats = get_stats(loader.storage)
assert {
@@ -119,9 +121,7 @@
"snapshot": 1,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="tar")
def test_check_revision_metadata_structure(swh_config, requests_mock_datadir):
@@ -131,6 +131,8 @@
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
+ assert_last_visit_ok(loader.storage, URL, status="full", type="tar")
+
expected_revision_id = hash_to_bytes("44183488c0774ce3c957fa19ba695cf18a4a42b3")
revision = list(loader.storage.revision_get([expected_revision_id]))[0]
@@ -166,6 +168,8 @@
actual_load_status["snapshot_id"] == _expected_new_snapshot_first_visit_id
) # noqa
+ assert_last_visit_ok(loader.storage, URL, status="full", type="tar")
+
stats = get_stats(loader.storage)
assert {
"content": len(_expected_new_contents_first_visit),
@@ -206,19 +210,15 @@
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "tar"
+
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
actual_load_status2 = loader.load()
assert actual_load_status2["status"] == "uneventful"
assert actual_load_status2["snapshot_id"] is not None
-
assert actual_load_status["snapshot_id"] == actual_load_status2["snapshot_id"]
- origin_visit2 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
urls = [
m.url
@@ -240,9 +240,7 @@
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
stats = get_stats(loader.storage)
assert {
@@ -295,9 +293,7 @@
"snapshot": 1 + 1,
} == stats2
- origin_visit2 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
urls = [
m.url
@@ -334,16 +330,12 @@
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
actual_load_status2 = loader.load()
assert actual_load_status2["status"] == "uneventful"
assert actual_load_status2["snapshot_id"] == actual_load_status["snapshot_id"]
- origin_visit2 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "tar"
+ assert_last_visit_ok(loader.storage, url, status="full", type="tar")
urls = [
m.url
diff --git a/swh/loader/package/cran/tests/test_cran.py b/swh/loader/package/cran/tests/test_cran.py
--- a/swh/loader/package/cran/tests/test_cran.py
+++ b/swh/loader/package/cran/tests/test_cran.py
@@ -20,7 +20,11 @@
from swh.core.tarball import uncompress
from swh.model.model import TimestampWithTimezone
-from swh.loader.package.tests.common import check_snapshot, get_stats
+from swh.loader.package.tests.common import (
+ check_snapshot,
+ get_stats,
+)
+from swh.loader.tests.conftest import assert_last_visit_ok
def test_cran_parse_date():
@@ -182,9 +186,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(origin_url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "cran"
+ assert_last_visit_ok(loader.storage, origin_url, status="full", type="cran")
visit_stats = get_stats(loader.storage)
assert {
@@ -241,9 +243,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(origin_url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "cran"
+ assert_last_visit_ok(loader.storage, origin_url, status="full", type="cran")
visit_stats = get_stats(loader.storage)
assert {
@@ -266,9 +266,7 @@
"snapshot_id": expected_snapshot_id,
}
- origin_visit2 = loader.storage.origin_visit_get_latest(origin_url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "cran"
+ assert_last_visit_ok(loader.storage, origin_url, status="full", type="cran")
visit_stats2 = get_stats(loader.storage)
visit_stats["origin_visit"] += 1
diff --git a/swh/loader/package/debian/tests/test_debian.py b/swh/loader/package/debian/tests/test_debian.py
--- a/swh/loader/package/debian/tests/test_debian.py
+++ b/swh/loader/package/debian/tests/test_debian.py
@@ -19,7 +19,12 @@
get_package_metadata,
extract_package,
)
-from swh.loader.package.tests.common import check_snapshot, get_stats
+from swh.loader.package.tests.common import (
+ check_snapshot,
+ get_stats,
+)
+from swh.loader.tests.conftest import assert_last_visit_ok
+
from swh.loader.package.debian.loader import resolve_revision_from
from swh.model.model import Person
@@ -100,10 +105,9 @@
"""With no prior visit, load a gnu project ends up with 1 snapshot
"""
+ url = "deb://Debian/packages/cicero"
loader = DebianLoader(
- url="deb://Debian/packages/cicero",
- date="2019-10-12T05:58:09.165557+00:00",
- packages=PACKAGE_PER_VERSION,
+ url=url, date="2019-10-12T05:58:09.165557+00:00", packages=PACKAGE_PER_VERSION,
)
actual_load_status = loader.load()
@@ -113,6 +117,8 @@
"snapshot_id": expected_snapshot_id,
}
+ assert_last_visit_ok(loader.storage, url, status="full", type="deb")
+
stats = get_stats(loader.storage)
assert {
"content": 42,
@@ -156,9 +162,7 @@
"snapshot_id": expected_snapshot_id,
}
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "deb"
+ assert_last_visit_ok(loader.storage, url, status="full", type="deb")
stats = get_stats(loader.storage)
assert {
@@ -188,9 +192,7 @@
# No change in between load
actual_load_status2 = loader.load()
assert actual_load_status2["status"] == "uneventful"
- origin_visit2 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "deb"
+ assert_last_visit_ok(loader.storage, url, status="full", type="deb")
stats2 = get_stats(loader.storage)
assert {
@@ -375,9 +377,7 @@
"snapshot_id": expected_snapshot_id,
}
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "deb"
+ assert_last_visit_ok(loader.storage, url, status="full", type="deb")
expected_snapshot = {
"id": expected_snapshot_id,
diff --git a/swh/loader/package/deposit/tests/test_deposit.py b/swh/loader/package/deposit/tests/test_deposit.py
--- a/swh/loader/package/deposit/tests/test_deposit.py
+++ b/swh/loader/package/deposit/tests/test_deposit.py
@@ -16,6 +16,7 @@
check_metadata_paths,
get_stats,
)
+from swh.loader.tests.conftest import assert_last_visit_ok
from swh.core.pytest_plugin import requests_mock_datadir_factory
@@ -91,6 +92,8 @@
assert actual_load_status["status"] == "uneventful"
assert actual_load_status["snapshot_id"] is not None
+ assert_last_visit_ok(loader.storage, url, status="partial", type="deposit")
+
stats = get_stats(loader.storage)
assert {
"content": 0,
@@ -104,10 +107,6 @@
"snapshot": 1,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "deposit"
-
def test_revision_metadata_structure(swh_config, requests_mock_datadir):
url = "https://hal-test.archives-ouvertes.fr/some-external-id"
@@ -154,6 +153,8 @@
"snapshot_id": expected_snapshot_id,
}
+ assert_last_visit_ok(loader.storage, url, status="full", type="deposit")
+
stats = get_stats(loader.storage)
assert {
"content": 303,
@@ -167,10 +168,6 @@
"snapshot": 1,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "deposit"
-
revision_id = "637318680351f5d78856d13264faebbd91efe9bb"
expected_branches = {
"HEAD": {"target": revision_id, "target_type": "revision",},
@@ -245,6 +242,7 @@
"status": "eventful",
"snapshot_id": expected_snapshot_id,
}
+ assert_last_visit_ok(loader.storage, url, status="full", type="deposit")
revision_id = "564d18943d71be80d0d73b43a77cfb205bcde96c"
expected_branches = {"HEAD": {"target": revision_id, "target_type": "revision"}}
@@ -255,12 +253,6 @@
check_snapshot(expected_snapshot, storage=loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
-
- # The visit is partial because some hash collision were detected
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "deposit"
-
raw_meta = loader.client.metadata_get(deposit_id)
# Ensure the date fields are set appropriately in the revision
diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py
--- a/swh/loader/package/loader.py
+++ b/swh/loader/package/loader.py
@@ -119,13 +119,17 @@
"""
return ""
- def last_snapshot(self) -> Optional[Snapshot]:
- """Retrieve the last snapshot
+ def last_snapshot(self, last_visit: Optional[Dict[str, Any]]) -> Optional[Snapshot]:
+ """Retrieve the last snapshot out of the last visit.
"""
+ if not last_visit:
+ return None
snapshot = None
- visit = self.storage.origin_visit_get_latest(self.url, require_snapshot=True)
- snapshot_id = None if not visit else visit.get("snapshot")
+ visit_status = self.storage.origin_visit_status_get_latest(
+ self.url, last_visit["visit"], require_snapshot=True,
+ )
+ snapshot_id = None if not visit_status else visit_status.snapshot
if snapshot_id is not None:
snapshot_dict = snapshot_get_all_branches(self.storage, snapshot_id)
if snapshot_dict:
@@ -303,6 +307,9 @@
result["snapshot_id"] = hash_to_hex(snapshot_id)
return result
+ # last visit if any
+ last_visit = self.storage.origin_visit_get_latest(self.url)
+
# Prepare origin and origin_visit
origin = Origin(url=self.url)
try:
@@ -324,7 +331,7 @@
return {"status": "failed"}
try:
- last_snapshot = self.last_snapshot()
+ last_snapshot = self.last_snapshot(last_visit)
logger.debug("last snapshot: %s", last_snapshot)
known_artifacts = self.known_artifacts(last_snapshot)
logger.debug("known artifacts: %s", known_artifacts)
diff --git a/swh/loader/package/nixguix/tests/test_nixguix.py b/swh/loader/package/nixguix/tests/test_nixguix.py
--- a/swh/loader/package/nixguix/tests/test_nixguix.py
+++ b/swh/loader/package/nixguix/tests/test_nixguix.py
@@ -22,10 +22,15 @@
clean_sources,
)
-from swh.loader.package.tests.common import get_stats, check_snapshot
+from swh.loader.package.tests.common import (
+ get_stats,
+ check_snapshot,
+)
from swh.loader.package.utils import download
from swh.model.hashutil import hash_to_bytes, hash_to_hex
from swh.storage.exc import HashCollision
+from swh.loader.tests.conftest import assert_last_visit_ok
+
sources_url = "https://nix-community.github.io/nixpkgs-swh/sources.json"
@@ -118,11 +123,9 @@
"snapshot": 1,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(sources_url)
# The visit is partial because urls pointing to non tarball file
# are not handled yet
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "nixguix"
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
def test_uncompress_failure(swh_config, requests_mock_datadir):
@@ -141,10 +144,9 @@
assert "https://example.com/file.txt" in urls
assert loader_status["status"] == "eventful"
- origin_visit = loader.storage.origin_visit_get_latest(sources_url)
# The visit is partial because urls pointing to non tarball files
# are not handled yet
- assert origin_visit["status"] == "partial"
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
def test_loader_incremental(swh_config, requests_mock_datadir):
@@ -158,6 +160,9 @@
loader.load()
expected_snapshot_id = "0c5881c74283793ebe9a09a105a9381e41380383"
assert load_status == {"status": "eventful", "snapshot_id": expected_snapshot_id}
+
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
+
expected_branches = {
"evaluation": {
"target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7",
@@ -200,6 +205,8 @@
expected_snapshot_id = "0c5881c74283793ebe9a09a105a9381e41380383"
assert load_status == {"status": "eventful", "snapshot_id": expected_snapshot_id}
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
+
expected_branches = {
"evaluation": {
"target": "cc4e04c26672dd74e5fd0fecb78b435fb55368f7",
@@ -236,6 +243,8 @@
expected_snapshot_id = "b0bfa75cbd0cc90aac3b9e95fb0f59c731176d97"
assert load_status == {"status": "eventful", "snapshot_id": expected_snapshot_id}
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
+
# This ensures visits are incremental. Indeed, if we request a
# second time an url, because of the requests_mock_datadir_visits
# fixture, the file has to end with `_visit1`.
@@ -294,6 +303,8 @@
res = loader.load()
assert res["status"] == "eventful"
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
+
expected_branches = {
"https://github.com/owner-1/repository-1/revision-1.tgz": {
"target": "488ad4e7b8e2511258725063cf43a2b897c503b4",
@@ -394,11 +405,8 @@
assert len(mock_download.mock_calls) == 2
- origin_visit = loader.storage.origin_visit_get_latest(sources_url)
-
- # The visit is partial because some hash collision were detected
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "nixguix"
+ # The visit is partial because some artifact downloads failed
+ assert_last_visit_ok(loader.storage, sources_url, status="partial", type="nixguix")
def test_load_nixguix_one_common_artifact_from_other_loader(
@@ -428,6 +436,8 @@
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] == expected_snapshot_id # noqa
+ assert_last_visit_ok(archive_loader.storage, gnu_url, status="full", type="tar")
+
gnu_snapshot = archive_loader.storage.snapshot_get(
hash_to_bytes(expected_snapshot_id)
)
@@ -456,6 +466,8 @@
actual_load_status2 = loader.load()
assert actual_load_status2["status"] == "eventful"
+ assert_last_visit_ok(loader.storage, sources_url, status="full", type="nixguix")
+
snapshot_id = actual_load_status2["snapshot_id"]
snapshot = loader.storage.snapshot_get(hash_to_bytes(snapshot_id))
snapshot.pop("next_branch") # snapshot_get endpoint detail to drop
@@ -486,6 +498,8 @@
assert last_snapshot.called
assert actual_load_status3["status"] == "eventful"
+ assert_last_visit_ok(loader.storage, sources_url, status="full", type="nixguix")
+
new_snapshot_id = "32ff641e510aceefc3a6d0dcbf208b2854d2e965"
assert actual_load_status3["snapshot_id"] == new_snapshot_id
diff --git a/swh/loader/package/npm/tests/test_npm.py b/swh/loader/package/npm/tests/test_npm.py
--- a/swh/loader/package/npm/tests/test_npm.py
+++ b/swh/loader/package/npm/tests/test_npm.py
@@ -20,6 +20,7 @@
check_metadata_paths,
get_stats,
)
+from swh.loader.tests.conftest import assert_last_visit_ok
def test_extract_npm_package_author(datadir):
@@ -350,9 +351,7 @@
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["status"] is not None
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="full", type="npm")
stats = get_stats(loader.storage)
@@ -375,9 +374,7 @@
assert snap_id2 is not None
assert snap_id2 != actual_load_status["snapshot_id"]
- origin_visit2 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit2["status"] == "full"
- assert origin_visit2["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="full", type="npm")
stats = get_stats(loader.storage)
@@ -410,9 +407,7 @@
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["status"] is not None
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="full", type="npm")
stats = get_stats(loader.storage)
@@ -524,9 +519,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="full", type="npm")
def test_npm_artifact_with_no_upload_time(swh_config, requests_mock_datadir):
@@ -547,9 +540,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="npm")
def test_npm_artifact_use_mtime_if_no_time(swh_config, requests_mock_datadir):
@@ -576,9 +567,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="full", type="npm")
def test_npm_no_artifact(swh_config, requests_mock_datadir):
@@ -593,6 +582,4 @@
"status": "failed",
}
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "npm"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="npm")
diff --git a/swh/loader/package/pypi/tests/test_pypi.py b/swh/loader/package/pypi/tests/test_pypi.py
--- a/swh/loader/package/pypi/tests/test_pypi.py
+++ b/swh/loader/package/pypi/tests/test_pypi.py
@@ -28,6 +28,7 @@
check_metadata_paths,
get_stats,
)
+from swh.loader.tests.conftest import assert_last_visit_ok
def test_author_basic():
@@ -226,9 +227,7 @@
"snapshot": 1,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="pypi")
# problem during loading:
@@ -260,9 +259,7 @@
"snapshot": 0,
} == stats
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="pypi")
# problem during loading: failure early enough in between swh contents...
@@ -389,9 +386,7 @@
}
check_snapshot(expected_snapshot, storage=loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "partial"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="partial", type="pypi")
def test_visit_with_1_release_artifact(swh_config, requests_mock_datadir):
@@ -476,9 +471,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
def test_multiple_visits_with_no_change(swh_config, requests_mock_datadir):
@@ -494,6 +487,7 @@
"status": "eventful",
"snapshot_id": snapshot_id,
}
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
stats = get_stats(loader.storage)
@@ -527,23 +521,23 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "pypi"
-
actual_load_status2 = loader.load()
assert actual_load_status2 == {
"status": "uneventful",
"snapshot_id": actual_load_status2["snapshot_id"],
}
+ visit_status2 = assert_last_visit_ok(
+ loader.storage, url, status="full", type="pypi"
+ )
+
stats2 = get_stats(loader.storage)
expected_stats2 = stats.copy()
expected_stats2["origin_visit"] = 1 + 1
assert expected_stats2 == stats2
# same snapshot
- actual_snapshot_id = origin_visit["snapshot"]
+ actual_snapshot_id = visit_status2.snapshot
assert actual_snapshot_id == hash_to_bytes(snapshot_id)
@@ -562,9 +556,7 @@
"snapshot_id": expected_snapshot_id,
}
- origin_visit1 = loader.storage.origin_visit_get_latest(url)
- assert origin_visit1["status"] == "full"
- assert origin_visit1["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
assert {
"content": 6,
@@ -591,10 +583,7 @@
"snapshot_id": expected_snapshot_id2,
}
- visits = list(loader.storage.origin_visit_get(url))
- assert len(visits) == 2
- assert visits[1]["status"] == "full"
- assert visits[1]["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
assert {
"content": 6 + 1, # 1 more content
@@ -674,9 +663,7 @@
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
urls = [
m.url
@@ -729,9 +716,7 @@
}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
def test_pypi_artifact_to_revision_id_none():
@@ -843,6 +828,4 @@
expected_snapshot = {"id": expected_snapshot_id, "branches": {}}
check_snapshot(expected_snapshot, loader.storage)
- origin_visit = loader.storage.origin_visit_get_latest(url)
- assert origin_visit["status"] == "full"
- assert origin_visit["type"] == "pypi"
+ assert_last_visit_ok(loader.storage, url, status="full", type="pypi")
diff --git a/swh/loader/package/tests/test_conftest.py b/swh/loader/package/tests/test_conftest.py
--- a/swh/loader/package/tests/test_conftest.py
+++ b/swh/loader/package/tests/test_conftest.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py
--- a/swh/loader/package/tests/test_loader.py
+++ b/swh/loader/package/tests/test_loader.py
@@ -10,8 +10,11 @@
def origin_add_one(self, origin):
raise ValueError("We refuse to add an origin")
+ def origin_visit_get_latest(self, origin):
+ return None
-class FakeStorage2:
+
+class FakeStorage2(FakeStorage):
def origin_add_one(self, origin):
return origin
diff --git a/swh/loader/tests/conftest.py b/swh/loader/tests/conftest.py
--- a/swh/loader/tests/conftest.py
+++ b/swh/loader/tests/conftest.py
@@ -1,11 +1,14 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import pytest
-from typing import Any, Dict
+from typing import Any, Dict, Optional
+
+from swh.model.model import OriginVisitStatus
+from swh.storage.algos.origin import origin_get_latest_visit_status
@pytest.fixture
@@ -17,3 +20,47 @@
"auth": {"username": "user", "password": "pass",},
},
}
+
+
+def assert_last_visit_ok(
+ storage,
+ url: str,
+ status: str,
+ type: Optional[str] = None,
+ snapshot: Optional[bytes] = None,
+) -> OriginVisitStatus:
+ """This retrieves the last visit and visit_status which are expected to exist.
+
+ This also checks that the {visit|visit_status} have their respective properties
+ correctly set.
+
+ This returns the last visit_status for that given origin.
+
+ Args:
+ url: Origin url
+ status: expected status
+ type: optional expected type to check against visit
+ snapshot: optional expected snapshot to check against visit status
+
+ Raises:
+ ValueError in case visit or visit status is not found
+ AssertionError if
+
+ Returns:
+ the visit status for further check during the remaining part of the test.
+
+ """
+ visit_and_status = origin_get_latest_visit_status(storage, url)
+ if not visit_and_status:
+ raise ValueError(f"Visit/visit_status should exist for origin {url}")
+ visit, visit_status = visit_and_status
+ if type:
+ assert visit.type == type, f"Visit with type {visit.type} instead of {type}"
+ assert (
+ visit_status.status == status
+ ), f"Visit_status with status {visit_status.status} instead of {status}"
+ if snapshot:
+ assert (
+ visit_status.snapshot == snapshot
+ ), f"Status with snapshot {visit_status.snapshot!r} instead of {snapshot!r}"
+ return visit_status