Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/nixguix/tests/test_nixguix.py
Show All 22 Lines | |||||
from swh.loader.tests import assert_last_visit_matches | from swh.loader.tests import assert_last_visit_matches | ||||
from swh.loader.tests import check_snapshot as check_snapshot_full | from swh.loader.tests import check_snapshot as check_snapshot_full | ||||
from swh.loader.tests import get_stats | from swh.loader.tests import get_stats | ||||
from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
ObjectType, | |||||
Person, | |||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
Release, | |||||
Snapshot, | Snapshot, | ||||
SnapshotBranch, | SnapshotBranch, | ||||
TargetType, | TargetType, | ||||
) | ) | ||||
from swh.model.swhids import ExtendedObjectType, ExtendedSWHID | from swh.model.swhids import ExtendedObjectType, ExtendedSWHID | ||||
from swh.storage.algos.origin import origin_get_latest_visit_status | from swh.storage.algos.origin import origin_get_latest_visit_status | ||||
from swh.storage.algos.snapshot import snapshot_get_all_branches | from swh.storage.algos.snapshot import snapshot_get_all_branches | ||||
from swh.storage.exc import HashCollision | from swh.storage.exc import HashCollision | ||||
Show All 9 Lines | with open( | ||||
datadir, "https_nix-community.github.io", "nixpkgs-swh_sources.json" | datadir, "https_nix-community.github.io", "nixpkgs-swh_sources.json" | ||||
), | ), | ||||
"rb", | "rb", | ||||
) as f: | ) as f: | ||||
return f.read() | return f.read() | ||||
SNAPSHOT1 = Snapshot( | SNAPSHOT1 = Snapshot( | ||||
id=hash_to_bytes("771d13ae4e799755c22d1e05da8fc39cf215de58"), | id=hash_to_bytes("efe5145f85af3fc87f34102d8b8481cd5198f4f8"), | ||||
branches={ | branches={ | ||||
b"evaluation": SnapshotBranch( | b"evaluation": SnapshotBranch( | ||||
target=hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), | target=hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), | ||||
target_type=TargetType.REVISION, | target_type=TargetType.REVISION, | ||||
), | ), | ||||
b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | ||||
target=hash_to_bytes("24853190589d26d0ea2b6c0330b553ff39176e0c"), | target=hash_to_bytes("df7811b9644ed8ef088e2e7add62ed32b0bab15f"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
def check_snapshot(snapshot: Snapshot, storage: StorageInterface): | def check_snapshot(snapshot: Snapshot, storage: StorageInterface): | ||||
# The `evaluation` branch is allowed to be unresolvable. It's possible at current | # The `evaluation` branch is allowed to be unresolvable. It's possible at current | ||||
▲ Show 20 Lines • Show All 195 Lines • ▼ Show 20 Lines | def test_clean_sources_unsupported_artifacts(swh_storage, requests_mock_datadir): | ||||
clean = clean_sources(sources, unsupported_file_extensions) | clean = clean_sources(sources, unsupported_file_extensions) | ||||
assert len(clean["sources"]) == len(supported_sources) | assert len(clean["sources"]) == len(supported_sources) | ||||
def test_loader_one_visit(swh_storage, requests_mock_datadir, raw_sources): | def test_loader_one_visit(swh_storage, requests_mock_datadir, raw_sources): | ||||
loader = NixGuixLoader(swh_storage, sources_url) | loader = NixGuixLoader(swh_storage, sources_url) | ||||
res = loader.load() | load_status = loader.load() | ||||
assert res["status"] == "eventful" | expected_snapshot_id_hex = "efe5145f85af3fc87f34102d8b8481cd5198f4f8" | ||||
expected_snapshot_id = hash_to_bytes(expected_snapshot_id_hex) | |||||
assert load_status == { | |||||
"status": "eventful", | |||||
"snapshot_id": expected_snapshot_id_hex, | |||||
} | |||||
release_id = hash_to_bytes("df7811b9644ed8ef088e2e7add62ed32b0bab15f") | |||||
expected_snapshot = Snapshot( | |||||
id=expected_snapshot_id, | |||||
branches={ | |||||
b"evaluation": SnapshotBranch( | |||||
target=hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), | |||||
target_type=TargetType.REVISION, | |||||
), | |||||
b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | |||||
target=release_id, target_type=TargetType.RELEASE, | |||||
), | |||||
}, | |||||
) | |||||
check_snapshot(expected_snapshot, storage=swh_storage) | |||||
assert swh_storage.release_get([release_id])[0] == Release( | |||||
id=release_id, | |||||
name=b"https://github.com/owner-1/repository-1/revision-1.tgz", | |||||
message=None, | |||||
target=hash_to_bytes("4de2e07d3742718d928e974b8a4c721b9f7b33bf"), | |||||
target_type=ObjectType.DIRECTORY, | |||||
synthetic=True, | |||||
author=Person.from_fullname(b""), | |||||
date=None, | |||||
) | |||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
"content": 1, | "content": 1, | ||||
"directory": 3, | "directory": 3, | ||||
"origin": 1, | "origin": 1, | ||||
"origin_visit": 1, | "origin_visit": 1, | ||||
"release": 1, | "release": 1, | ||||
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | assert { | ||||
"release": 1, | "release": 1, | ||||
"revision": 0, | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
loader = NixGuixLoader(swh_storage, sources_url) | loader = NixGuixLoader(swh_storage, sources_url) | ||||
load_status = loader.load() | load_status = loader.load() | ||||
expected_snapshot_id_hex = "c5bba84fd5ac3342566effb86190619092d34e79" | expected_snapshot_id_hex = "c1983a0a3f647548e1fb92f30339da6848fe9f7a" | ||||
expected_snapshot_id = hash_to_bytes(expected_snapshot_id_hex) | expected_snapshot_id = hash_to_bytes(expected_snapshot_id_hex) | ||||
assert load_status == { | assert load_status == { | ||||
"status": "eventful", | "status": "eventful", | ||||
"snapshot_id": expected_snapshot_id_hex, | "snapshot_id": expected_snapshot_id_hex, | ||||
} | } | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
Show All 9 Lines | def test_loader_two_visits(swh_storage, requests_mock_datadir_visits): | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=expected_snapshot_id, | id=expected_snapshot_id, | ||||
branches={ | branches={ | ||||
b"evaluation": SnapshotBranch( | b"evaluation": SnapshotBranch( | ||||
target=hash_to_bytes("602140776b2ce6c9159bcf52ada73a297c063d5e"), | target=hash_to_bytes("602140776b2ce6c9159bcf52ada73a297c063d5e"), | ||||
target_type=TargetType.REVISION, | target_type=TargetType.REVISION, | ||||
), | ), | ||||
b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | ||||
target=hash_to_bytes("24853190589d26d0ea2b6c0330b553ff39176e0c"), | target=hash_to_bytes("df7811b9644ed8ef088e2e7add62ed32b0bab15f"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"https://github.com/owner-2/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-2/repository-1/revision-1.tgz": SnapshotBranch( | ||||
target=hash_to_bytes("3d44fbe814ba802cfd77f83975e45766d3a2ba85"), | target=hash_to_bytes("5cc0115cd643902b837cb6cfbc9f5865bc5a7cb2"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
check_snapshot(expected_snapshot, storage=swh_storage) | check_snapshot(expected_snapshot, storage=swh_storage) | ||||
stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
assert { | assert { | ||||
▲ Show 20 Lines • Show All 113 Lines • ▼ Show 20 Lines | gnu_artifacts = [ | ||||
"url": artifact_url, | "url": artifact_url, | ||||
"length": 221837, | "length": 221837, | ||||
"filename": f"8sync-{release}.tar.gz", | "filename": f"8sync-{release}.tar.gz", | ||||
"version": release, | "version": release, | ||||
} | } | ||||
] | ] | ||||
archive_loader = ArchiveLoader(swh_storage, url=gnu_url, artifacts=gnu_artifacts) | archive_loader = ArchiveLoader(swh_storage, url=gnu_url, artifacts=gnu_artifacts) | ||||
actual_load_status = archive_loader.load() | actual_load_status = archive_loader.load() | ||||
expected_snapshot_id = "cdf8f335fa0c81c8ad089870ec14f52b1980eb6c" | expected_snapshot_id = "af62f6f6d464f9b29f270d1bbefa355af38946c4" | ||||
assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
assert actual_load_status["snapshot_id"] == expected_snapshot_id # noqa | assert actual_load_status["snapshot_id"] == expected_snapshot_id # noqa | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
archive_loader.storage, | archive_loader.storage, | ||||
gnu_url, | gnu_url, | ||||
status="full", | status="full", | ||||
type="tar", | type="tar", | ||||
Show All 37 Lines |