Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/archive/tests/test_archive.py
Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | _expected_new_directories_first_visit = [ | ||||
"263be23b4a8101d3ad0d9831319a3e0f2b065f36", | "263be23b4a8101d3ad0d9831319a3e0f2b065f36", | ||||
"7f6e63ba6eb3e2236f65892cd822041f1a01dd5c", | "7f6e63ba6eb3e2236f65892cd822041f1a01dd5c", | ||||
"4db0a3ecbc976083e2dac01a62f93729698429a3", | "4db0a3ecbc976083e2dac01a62f93729698429a3", | ||||
"dfef1c80e1098dd5deda664bb44a9ab1f738af13", | "dfef1c80e1098dd5deda664bb44a9ab1f738af13", | ||||
"eca971d346ea54d95a6e19d5051f900237fafdaa", | "eca971d346ea54d95a6e19d5051f900237fafdaa", | ||||
"3aebc29ed1fccc4a6f2f2010fb8e57882406b528", | "3aebc29ed1fccc4a6f2f2010fb8e57882406b528", | ||||
] | ] | ||||
_expected_new_revisions_first_visit = { | _expected_new_releases_first_visit = { | ||||
"44183488c0774ce3c957fa19ba695cf18a4a42b3": ( | "c9786c1e3b46f52779c727d3509d66ebf8948d88": ( | ||||
"3aebc29ed1fccc4a6f2f2010fb8e57882406b528" | "3aebc29ed1fccc4a6f2f2010fb8e57882406b528" | ||||
) | ) | ||||
} | } | ||||
def test_archive_visit_with_no_artifact_found(swh_storage, requests_mock_datadir): | def test_archive_visit_with_no_artifact_found(swh_storage, requests_mock_datadir): | ||||
url = URL | url = URL | ||||
unknown_artifact_url = "https://ftp.g.o/unknown/8sync-0.1.0.tar.gz" | unknown_artifact_url = "https://ftp.g.o/unknown/8sync-0.1.0.tar.gz" | ||||
Show All 37 Lines | ): | ||||
""" | """ | ||||
loader = ArchiveLoader(swh_storage, URL, artifacts=GNU_ARTIFACTS[:1]) | loader = ArchiveLoader(swh_storage, URL, artifacts=GNU_ARTIFACTS[:1]) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
expected_snapshot_first_visit_id = hash_to_bytes( | expected_snapshot_first_visit_id = hash_to_bytes( | ||||
"c419397fd912039825ebdbea378bc6283f006bf5" | "cdf8f335fa0c81c8ad089870ec14f52b1980eb6c" | ||||
) | ) | ||||
assert ( | assert ( | ||||
hash_to_bytes(actual_load_status["snapshot_id"]) | hash_to_bytes(actual_load_status["snapshot_id"]) | ||||
== expected_snapshot_first_visit_id | == expected_snapshot_first_visit_id | ||||
) | ) | ||||
assert_last_visit_matches(swh_storage, URL, status="full", type="tar") | assert_last_visit_matches(swh_storage, URL, status="full", type="tar") | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": len(_expected_new_contents_first_visit), | "content": len(_expected_new_contents_first_visit), | ||||
"directory": len(_expected_new_directories_first_visit), | "directory": len(_expected_new_directories_first_visit), | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1, | "origin_visit": 1, | ||||
"release": 0, | "release": len(_expected_new_releases_first_visit), | ||||
"revision": len(_expected_new_revisions_first_visit), | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=expected_snapshot_first_visit_id, | id=expected_snapshot_first_visit_id, | ||||
branches={ | branches={ | ||||
b"HEAD": SnapshotBranch( | b"HEAD": SnapshotBranch( | ||||
target_type=TargetType.ALIAS, target=b"releases/0.1.0", | target_type=TargetType.ALIAS, target=b"releases/0.1.0", | ||||
), | ), | ||||
b"releases/0.1.0": SnapshotBranch( | b"releases/0.1.0": SnapshotBranch( | ||||
target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
target=hash_to_bytes(list(_expected_new_revisions_first_visit)[0]), | target=hash_to_bytes(list(_expected_new_releases_first_visit)[0]), | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
check_snapshot(expected_snapshot, swh_storage) | check_snapshot(expected_snapshot, swh_storage) | ||||
expected_contents = map(hash_to_bytes, _expected_new_contents_first_visit) | expected_contents = map(hash_to_bytes, _expected_new_contents_first_visit) | ||||
assert list(swh_storage.content_missing_per_sha1(expected_contents)) == [] | assert list(swh_storage.content_missing_per_sha1(expected_contents)) == [] | ||||
expected_dirs = map(hash_to_bytes, _expected_new_directories_first_visit) | expected_dirs = map(hash_to_bytes, _expected_new_directories_first_visit) | ||||
assert list(swh_storage.directory_missing(expected_dirs)) == [] | assert list(swh_storage.directory_missing(expected_dirs)) == [] | ||||
expected_revs = map(hash_to_bytes, _expected_new_revisions_first_visit) | expected_rels = map(hash_to_bytes, _expected_new_releases_first_visit) | ||||
assert list(swh_storage.revision_missing(expected_revs)) == [] | assert list(swh_storage.release_missing(expected_rels)) == [] | ||||
def test_archive_2_visits_without_change(swh_storage, requests_mock_datadir): | def test_archive_2_visits_without_change(swh_storage, requests_mock_datadir): | ||||
"""With no prior visit, load a gnu project ends up with 1 snapshot | """With no prior visit, load a gnu project ends up with 1 snapshot | ||||
""" | """ | ||||
url = URL | url = URL | ||||
loader = ArchiveLoader(swh_storage, url, artifacts=GNU_ARTIFACTS[:1]) | loader = ArchiveLoader(swh_storage, url, artifacts=GNU_ARTIFACTS[:1]) | ||||
Show All 34 Lines | def test_archive_2_visits_with_new_artifact(swh_storage, requests_mock_datadir): | ||||
assert_last_visit_matches(swh_storage, url, status="full", type="tar") | assert_last_visit_matches(swh_storage, url, status="full", type="tar") | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": len(_expected_new_contents_first_visit), | "content": len(_expected_new_contents_first_visit), | ||||
"directory": len(_expected_new_directories_first_visit), | "directory": len(_expected_new_directories_first_visit), | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1, | "origin_visit": 1, | ||||
"release": 0, | "release": len(_expected_new_releases_first_visit), | ||||
"revision": len(_expected_new_revisions_first_visit), | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
urls = [ | urls = [ | ||||
m.url | m.url | ||||
for m in requests_mock_datadir.request_history | for m in requests_mock_datadir.request_history | ||||
if m.url.startswith("https://ftp.gnu.org") | if m.url.startswith("https://ftp.gnu.org") | ||||
Show All 11 Lines | def test_archive_2_visits_with_new_artifact(swh_storage, requests_mock_datadir): | ||||
assert actual_load_status2["snapshot_id"] is not None | assert actual_load_status2["snapshot_id"] is not None | ||||
stats2 = get_stats(swh_storage) | stats2 = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": len(_expected_new_contents_first_visit) + 14, | "content": len(_expected_new_contents_first_visit) + 14, | ||||
"directory": len(_expected_new_directories_first_visit) + 8, | "directory": len(_expected_new_directories_first_visit) + 8, | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1 + 1, | "origin_visit": 1 + 1, | ||||
"release": 0, | "release": len(_expected_new_releases_first_visit) + 1, | ||||
"revision": len(_expected_new_revisions_first_visit) + 1, | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1 + 1, | "snapshot": 1 + 1, | ||||
} == stats2 | } == stats2 | ||||
assert_last_visit_matches(swh_storage, url, status="full", type="tar") | assert_last_visit_matches(swh_storage, url, status="full", type="tar") | ||||
urls = [ | urls = [ | ||||
m.url | m.url | ||||
▲ Show 20 Lines • Show All 198 Lines • Show Last 20 Lines |