Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/debian/tests/test_debian.py
Show First 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | def test_debian_first_visit(swh_storage, requests_mock_datadir): | ||||
loader = DebianLoader( | loader = DebianLoader( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
date="2019-10-12T05:58:09.165557+00:00", | date="2019-10-12T05:58:09.165557+00:00", | ||||
packages=PACKAGE_PER_VERSION, | packages=PACKAGE_PER_VERSION, | ||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = "3b6b66e6ee4e7d903a379a882684a2a50480c0b4" | expected_snapshot_id = "8bc5d12e2443ab216fdd2f969b25b39e96c20fef" | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
status="full", | status="full", | ||||
type="deb", | type="deb", | ||||
snapshot=hash_to_bytes(expected_snapshot_id), | snapshot=hash_to_bytes(expected_snapshot_id), | ||||
) | ) | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=hash_to_bytes(expected_snapshot_id), | id=hash_to_bytes(expected_snapshot_id), | ||||
branches={ | branches={ | ||||
b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | ||||
target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), | target=hash_to_bytes("5a99736512d381700c5f54d7fdd6b46e136535a2"), | ||||
) | ) | ||||
}, | }, | ||||
) # different than the previous loader as no release is done | ) # different than the previous loader as no release is done | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": 42, | "content": 42, | ||||
"directory": 2, | "directory": 2, | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1, | "origin_visit": 1, | ||||
"release": 0, | "release": 1, # all artifacts under 1 release | ||||
"revision": 1, # all artifacts under 1 revision | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
def test_debian_first_visit_then_another_visit(swh_storage, requests_mock_datadir): | def test_debian_first_visit_then_another_visit(swh_storage, requests_mock_datadir): | ||||
"""With no prior visit, load a debian project ends up with 1 snapshot | """With no prior visit, load a debian project ends up with 1 snapshot | ||||
""" | """ | ||||
loader = DebianLoader( | loader = DebianLoader( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
date="2019-10-12T05:58:09.165557+00:00", | date="2019-10-12T05:58:09.165557+00:00", | ||||
packages=PACKAGE_PER_VERSION, | packages=PACKAGE_PER_VERSION, | ||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = "3b6b66e6ee4e7d903a379a882684a2a50480c0b4" | expected_snapshot_id = "8bc5d12e2443ab216fdd2f969b25b39e96c20fef" | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
status="full", | status="full", | ||||
type="deb", | type="deb", | ||||
snapshot=hash_to_bytes(expected_snapshot_id), | snapshot=hash_to_bytes(expected_snapshot_id), | ||||
) | ) | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=hash_to_bytes(expected_snapshot_id), | id=hash_to_bytes(expected_snapshot_id), | ||||
branches={ | branches={ | ||||
b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | ||||
target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), | target=hash_to_bytes("5a99736512d381700c5f54d7fdd6b46e136535a2"), | ||||
) | ) | ||||
}, | }, | ||||
) # different than the previous loader as no release is done | ) # different than the previous loader as no release is done | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": 42, | "content": 42, | ||||
"directory": 2, | "directory": 2, | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1, | "origin_visit": 1, | ||||
"release": 0, | "release": 1, # all artifacts under 1 release | ||||
"revision": 1, # all artifacts under 1 revision | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
# No change in between load | # No change in between load | ||||
actual_load_status2 = loader.load() | actual_load_status2 = loader.load() | ||||
assert actual_load_status2["status"] == "uneventful" | assert actual_load_status2["status"] == "uneventful" | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
status="full", | status="full", | ||||
type="deb", | type="deb", | ||||
snapshot=hash_to_bytes(expected_snapshot_id), | snapshot=hash_to_bytes(expected_snapshot_id), | ||||
) | ) | ||||
stats2 = get_stats(swh_storage) | stats2 = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": 42 + 0, | "content": 42 + 0, | ||||
"directory": 2 + 0, | "directory": 2 + 0, | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1 + 1, # a new visit occurred | "origin_visit": 1 + 1, # a new visit occurred | ||||
"release": 0, | "release": 1, | ||||
"revision": 1, | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, # same snapshot across 2 visits | "snapshot": 1, # same snapshot across 2 visits | ||||
} == stats2 | } == stats2 | ||||
urls = [ | urls = [ | ||||
m.url | m.url | ||||
for m in requests_mock_datadir.request_history | for m in requests_mock_datadir.request_history | ||||
if m.url.startswith("http://deb.debian.org") | if m.url.startswith("http://deb.debian.org") | ||||
▲ Show 20 Lines • Show All 183 Lines • ▼ Show 20 Lines | def test_debian_multiple_packages(swh_storage, requests_mock_datadir): | ||||
loader = DebianLoader( | loader = DebianLoader( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
date="2019-10-12T05:58:09.165557+00:00", | date="2019-10-12T05:58:09.165557+00:00", | ||||
packages=PACKAGES_PER_VERSION, | packages=PACKAGES_PER_VERSION, | ||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
expected_snapshot_id = "defc19021187f3727293121fcf6c5c82cb923604" | expected_snapshot_id = "3d26243c91eb084c350627a5a102cfe039c5b92a" | ||||
assert actual_load_status == { | assert actual_load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id, | "snapshot_id": expected_snapshot_id, | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
URL, | URL, | ||||
status="full", | status="full", | ||||
type="deb", | type="deb", | ||||
snapshot=hash_to_bytes(expected_snapshot_id), | snapshot=hash_to_bytes(expected_snapshot_id), | ||||
) | ) | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=hash_to_bytes(expected_snapshot_id), | id=hash_to_bytes(expected_snapshot_id), | ||||
branches={ | branches={ | ||||
b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | b"releases/stretch/contrib/0.7.2-3": SnapshotBranch( | ||||
target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("2807f5b3f84368b4889a9ae827fe85854ffecf07"), | target=hash_to_bytes("5a99736512d381700c5f54d7fdd6b46e136535a2"), | ||||
), | ), | ||||
b"releases/buster/contrib/0.7.2-4": SnapshotBranch( | b"releases/buster/contrib/0.7.2-4": SnapshotBranch( | ||||
target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes("8224139c274c984147ef4b09aa0e462c55a10bd3"), | target=hash_to_bytes("192fc7ccce80f64a0d3cf33d379133af067ec721"), | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) |