Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/nixguix/tests/test_nixguix.py
| Show First 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | with open( | ||||
| datadir, "https_nix-community.github.io", "nixpkgs-swh_sources.json" | datadir, "https_nix-community.github.io", "nixpkgs-swh_sources.json" | ||||
| ), | ), | ||||
| "rb", | "rb", | ||||
| ) as f: | ) as f: | ||||
| return f.read() | return f.read() | ||||
| SNAPSHOT1 = Snapshot( | SNAPSHOT1 = Snapshot( | ||||
| id=hash_to_bytes("0c5881c74283793ebe9a09a105a9381e41380383"), | id=hash_to_bytes("771d13ae4e799755c22d1e05da8fc39cf215de58"), | ||||
| branches={ | branches={ | ||||
| b"evaluation": SnapshotBranch( | b"evaluation": SnapshotBranch( | ||||
| target=hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), | target=hash_to_bytes("cc4e04c26672dd74e5fd0fecb78b435fb55368f7"), | ||||
| target_type=TargetType.REVISION, | target_type=TargetType.REVISION, | ||||
| ), | ), | ||||
| b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | ||||
| target=hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), | target=hash_to_bytes("24853190589d26d0ea2b6c0330b553ff39176e0c"), | ||||
| target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
| ), | ), | ||||
| }, | }, | ||||
| ) | ) | ||||
| def check_snapshot(snapshot: Snapshot, storage: StorageInterface): | def check_snapshot(snapshot: Snapshot, storage: StorageInterface): | ||||
| # The `evaluation` branch is allowed to be unresolvable. It's possible at current | # The `evaluation` branch is allowed to be unresolvable. It's possible at current | ||||
| # nixguix visit time, it is not yet visited (the git loader is in charge of its | # nixguix visit time, it is not yet visited (the git loader is in charge of its | ||||
| ▲ Show 20 Lines • Show All 203 Lines • ▼ Show 20 Lines | def test_loader_one_visit(swh_storage, requests_mock_datadir, raw_sources): | ||||
| assert res["status"] == "eventful" | assert res["status"] == "eventful" | ||||
| stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
| assert { | assert { | ||||
| "content": 1, | "content": 1, | ||||
| "directory": 3, | "directory": 3, | ||||
| "origin": 1, | "origin": 1, | ||||
| "origin_visit": 1, | "origin_visit": 1, | ||||
| "release": 0, | "release": 1, | ||||
| "revision": 1, | "revision": 0, | ||||
| "skipped_content": 0, | "skipped_content": 0, | ||||
| "snapshot": 1, | "snapshot": 1, | ||||
| } == stats | } == stats | ||||
| # The visit is partial because urls pointing to non tarball file | # The visit is partial because urls pointing to non tarball file | ||||
| # are not handled yet | # are not handled yet | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, sources_url, status="partial", type="nixguix" | swh_storage, sources_url, status="partial", type="nixguix" | ||||
| ▲ Show 20 Lines • Show All 105 Lines • ▼ Show 20 Lines | def test_loader_two_visits(swh_storage, requests_mock_datadir_visits): | ||||
| check_snapshot(SNAPSHOT1, storage=swh_storage) | check_snapshot(SNAPSHOT1, storage=swh_storage) | ||||
| stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
| assert { | assert { | ||||
| "content": 1, | "content": 1, | ||||
| "directory": 3, | "directory": 3, | ||||
| "origin": 1, | "origin": 1, | ||||
| "origin_visit": 1, | "origin_visit": 1, | ||||
| "release": 0, | "release": 1, | ||||
| "revision": 1, | "revision": 0, | ||||
| "skipped_content": 0, | "skipped_content": 0, | ||||
| "snapshot": 1, | "snapshot": 1, | ||||
| } == stats | } == stats | ||||
| loader = NixGuixLoader(swh_storage, sources_url) | loader = NixGuixLoader(swh_storage, sources_url) | ||||
| load_status = loader.load() | load_status = loader.load() | ||||
| expected_snapshot_id_hex = "b0bfa75cbd0cc90aac3b9e95fb0f59c731176d97" | expected_snapshot_id_hex = "c5bba84fd5ac3342566effb86190619092d34e79" | ||||
| expected_snapshot_id = hash_to_bytes(expected_snapshot_id_hex) | expected_snapshot_id = hash_to_bytes(expected_snapshot_id_hex) | ||||
| assert load_status == { | assert load_status == { | ||||
| "status": "eventful", | "status": "eventful", | ||||
| "snapshot_id": expected_snapshot_id_hex, | "snapshot_id": expected_snapshot_id_hex, | ||||
| } | } | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| swh_storage, | swh_storage, | ||||
| Show All 9 Lines | def test_loader_two_visits(swh_storage, requests_mock_datadir_visits): | ||||
| expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
| id=expected_snapshot_id, | id=expected_snapshot_id, | ||||
| branches={ | branches={ | ||||
| b"evaluation": SnapshotBranch( | b"evaluation": SnapshotBranch( | ||||
| target=hash_to_bytes("602140776b2ce6c9159bcf52ada73a297c063d5e"), | target=hash_to_bytes("602140776b2ce6c9159bcf52ada73a297c063d5e"), | ||||
| target_type=TargetType.REVISION, | target_type=TargetType.REVISION, | ||||
| ), | ), | ||||
| b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-1/repository-1/revision-1.tgz": SnapshotBranch( | ||||
| target=hash_to_bytes("488ad4e7b8e2511258725063cf43a2b897c503b4"), | target=hash_to_bytes("24853190589d26d0ea2b6c0330b553ff39176e0c"), | ||||
| target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
| ), | ), | ||||
| b"https://github.com/owner-2/repository-1/revision-1.tgz": SnapshotBranch( | b"https://github.com/owner-2/repository-1/revision-1.tgz": SnapshotBranch( | ||||
| target=hash_to_bytes("85e0bad74e33e390aaeb74f139853ae3863ee544"), | target=hash_to_bytes("3d44fbe814ba802cfd77f83975e45766d3a2ba85"), | ||||
| target_type=TargetType.REVISION, | target_type=TargetType.RELEASE, | ||||
| ), | ), | ||||
| }, | }, | ||||
| ) | ) | ||||
| check_snapshot(expected_snapshot, storage=swh_storage) | check_snapshot(expected_snapshot, storage=swh_storage) | ||||
| stats = get_stats(swh_storage) | stats = get_stats(swh_storage) | ||||
| assert { | assert { | ||||
| "content": 2, | "content": 2, | ||||
| "directory": 5, | "directory": 5, | ||||
| "origin": 1, | "origin": 1, | ||||
| "origin_visit": 2, | "origin_visit": 2, | ||||
| "release": 0, | "release": 2, | ||||
| "revision": 2, | "revision": 0, | ||||
| "skipped_content": 0, | "skipped_content": 0, | ||||
| "snapshot": 2, | "snapshot": 2, | ||||
| } == stats | } == stats | ||||
| def test_evaluation_branch(swh_storage, requests_mock_datadir): | def test_evaluation_branch(swh_storage, requests_mock_datadir): | ||||
| loader = NixGuixLoader(swh_storage, sources_url) | loader = NixGuixLoader(swh_storage, sources_url) | ||||
| res = loader.load() | res = loader.load() | ||||
| ▲ Show 20 Lines • Show All 99 Lines • ▼ Show 20 Lines | gnu_artifacts = [ | ||||
| "url": artifact_url, | "url": artifact_url, | ||||
| "length": 221837, | "length": 221837, | ||||
| "filename": f"8sync-{release}.tar.gz", | "filename": f"8sync-{release}.tar.gz", | ||||
| "version": release, | "version": release, | ||||
| } | } | ||||
| ] | ] | ||||
| archive_loader = ArchiveLoader(swh_storage, url=gnu_url, artifacts=gnu_artifacts) | archive_loader = ArchiveLoader(swh_storage, url=gnu_url, artifacts=gnu_artifacts) | ||||
| actual_load_status = archive_loader.load() | actual_load_status = archive_loader.load() | ||||
| expected_snapshot_id = "c419397fd912039825ebdbea378bc6283f006bf5" | expected_snapshot_id = "cdf8f335fa0c81c8ad089870ec14f52b1980eb6c" | ||||
| assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
| assert actual_load_status["snapshot_id"] == expected_snapshot_id # noqa | assert actual_load_status["snapshot_id"] == expected_snapshot_id # noqa | ||||
| assert_last_visit_matches( | assert_last_visit_matches( | ||||
| archive_loader.storage, | archive_loader.storage, | ||||
| gnu_url, | gnu_url, | ||||
| status="full", | status="full", | ||||
| type="tar", | type="tar", | ||||
| Show All 37 Lines | |||||