Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/crates/tests/test_crates.py
# Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import json | |||||
from pathlib import Path | |||||
import pytest | import pytest | ||||
from swh.loader.package import __version__ | |||||
from swh.loader.package.crates.loader import CratesLoader | from swh.loader.package.crates.loader import CratesLoader | ||||
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | ||||
from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | |||||
MetadataAuthorityType, | |||||
MetadataFetcher, | |||||
ObjectType, | ObjectType, | ||||
Person, | Person, | ||||
RawExtrinsicMetadata, | |||||
Release, | Release, | ||||
Snapshot, | Snapshot, | ||||
SnapshotBranch, | SnapshotBranch, | ||||
TargetType, | TargetType, | ||||
TimestampWithTimezone, | TimestampWithTimezone, | ||||
) | ) | ||||
from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID | |||||
from swh.model.swhids import ObjectType as OType | |||||
from swh.storage.interface import PagedResult | |||||
CRATES_EXTRA = [ | |||||
{ | @pytest.fixture | ||||
"url": "https://crates.io/api/v1/crates/hg-core", | def expected(datadir): | ||||
"artifacts": [ | fp = datadir / Path("expected.json") | ||||
{ | return json.loads(fp.read_bytes()) | ||||
"checksums": { | |||||
"sha256": "48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d", # noqa: B950 | |||||
}, | |||||
"filename": "hg-core-0.0.1.crate", | |||||
"url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate", | |||||
"version": "0.0.1", | |||||
}, | |||||
], | |||||
}, | |||||
{ | |||||
"url": "https://crates.io/api/v1/crates/micro-timer", | |||||
"artifacts": [ | |||||
{ | |||||
"checksums": { | |||||
"sha256": "69ad8fd116f8af0298ae4e83e587b1600af12709022471e25581c3aeb1da77ce", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.1.0.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.0.crate", | |||||
"version": "0.1.0", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "7b3f65fe0e109daad8d47e1938c9b5f9353efacd86bbe7ff013f84ae7ca758bf", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.1.1.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.1.crate", | |||||
"version": "0.1.1", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "16439fea388f712c1df7737ceb8f784d407844624b4796faf1e1bf8bbaa97445", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.1.2.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.2.crate", | |||||
"version": "0.1.2", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "336b4c0f071d16674747faa4643d742cc096fec2bf8cf01bb1a98d984bedcaf1", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.2.0.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.0.crate", | |||||
"version": "0.2.0", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "987429cd6162a80ed5ff44fc790f5090b1c6d617ac73a2e272965ed91201d79b", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.2.1.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.1.crate", | |||||
"version": "0.2.1", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "25b31d6cb9112984323d05d7a353f272ae5d7a307074f9ab9b25c00121b8c947", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.3.0.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.0.crate", | |||||
"version": "0.3.0", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "2620153e1d903d26b72b89f0e9c48d8c4756cba941c185461dddc234980c298c", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.3.1.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.1.crate", | |||||
"version": "0.3.1", | |||||
}, | |||||
{ | |||||
"checksums": { | |||||
"sha256": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", # noqa: B950 | |||||
}, | |||||
"filename": "micro-timer-0.4.0.crate", | |||||
"url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate", | |||||
"version": "0.4.0", | |||||
}, | |||||
], | |||||
}, | |||||
] | |||||
def test_get_versions(requests_mock_datadir, swh_storage): | def test_get_versions(requests_mock_datadir, swh_storage, expected): | ||||
loader = CratesLoader( | loader = CratesLoader( | ||||
swh_storage, | swh_storage, | ||||
url=CRATES_EXTRA[1]["url"], | url=expected[1]["url"], | ||||
artifacts=CRATES_EXTRA[1]["artifacts"], | artifacts=expected[1]["artifacts"], | ||||
crates_metadata=expected[1]["crates_metadata"], | |||||
) | ) | ||||
assert loader.get_versions() == [ | assert loader.get_versions() == [ | ||||
"0.1.0", | "0.1.0", | ||||
"0.1.1", | "0.1.1", | ||||
"0.1.2", | "0.1.2", | ||||
"0.2.0", | "0.2.0", | ||||
"0.2.1", | "0.2.1", | ||||
"0.3.0", | "0.3.0", | ||||
"0.3.1", | "0.3.1", | ||||
"0.4.0", | "0.4.0", | ||||
] | ] | ||||
def test_get_default_version(requests_mock_datadir, swh_storage): | def test_get_default_version(requests_mock_datadir, swh_storage, expected): | ||||
loader = CratesLoader( | loader = CratesLoader( | ||||
swh_storage, | swh_storage, | ||||
url=CRATES_EXTRA[1]["url"], | url=expected[1]["url"], | ||||
artifacts=CRATES_EXTRA[1]["artifacts"], | artifacts=expected[1]["artifacts"], | ||||
crates_metadata=expected[1]["crates_metadata"], | |||||
) | ) | ||||
assert loader.get_default_version() == "0.4.0" | assert loader.get_default_version() == "0.4.0" | ||||
def test_crate_invalid_origin_archive_not_found(swh_storage, requests_mock_datadir): | def test_crate_invalid_origin_archive_not_found(swh_storage, requests_mock_datadir): | ||||
url = "https://nowhere-to-run/nowhere-to-hide" | url = "https://nowhere-to-run/nowhere-to-hide" | ||||
loader = CratesLoader( | loader = CratesLoader( | ||||
swh_storage, | swh_storage, | ||||
url, | url, | ||||
artifacts=[ | artifacts=[ | ||||
{ | { | ||||
"version": "0.0.1", | |||||
"filename": "nowhere-to-hide-0.0.1.crate", | "filename": "nowhere-to-hide-0.0.1.crate", | ||||
"url": "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate", | "url": "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate", | ||||
"checksums": { | |||||
"sha256": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", # noqa: B950 | |||||
}, | |||||
}, | |||||
], | |||||
crates_metadata=[ | |||||
{ | |||||
"version": "0.0.1", | "version": "0.0.1", | ||||
"yanked": True, | |||||
"last_update": "1970-01-01T00:00:00.000000+00:00", | |||||
}, | }, | ||||
], | ], | ||||
) | ) | ||||
with pytest.raises(Exception): | with pytest.raises(Exception): | ||||
assert loader.load() == {"status": "failed"} | assert loader.load() == {"status": "failed"} | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, url, status="not_found", type="crates", snapshot=None | swh_storage, url, status="not_found", type="crates", snapshot=None | ||||
) | ) | ||||
def test_crates_loader_load_one_version(datadir, requests_mock_datadir, swh_storage): | def test_crates_loader_load_one_version( | ||||
datadir, requests_mock_datadir, swh_storage, expected | |||||
): | |||||
loader = CratesLoader( | loader = CratesLoader( | ||||
swh_storage, | swh_storage, | ||||
url=CRATES_EXTRA[0]["url"], | url=expected[0]["url"], | ||||
artifacts=CRATES_EXTRA[0]["artifacts"], | artifacts=expected[0]["artifacts"], | ||||
crates_metadata=expected[0]["crates_metadata"], | |||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
assert actual_load_status["snapshot_id"] is not None | assert actual_load_status["snapshot_id"] is not None | ||||
expected_snapshot_id = "b3affb4949eb89b244f0e1d1fe235fc1d26bde76" | expected_snapshot_id = "779fa94645a772fb9992c2bf80bf5e2ecfdb1b5c" | ||||
expected_release_id = "237c4cdd44a90e620795e5a07ebcc72bc82487f7" | expected_release_id = "bb6f9b125867a8b4fa0b2febf890a317744e0140" | ||||
assert expected_snapshot_id == actual_load_status["snapshot_id"] | assert expected_snapshot_id == actual_load_status["snapshot_id"] | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=hash_to_bytes(actual_load_status["snapshot_id"]), | id=hash_to_bytes(actual_load_status["snapshot_id"]), | ||||
branches={ | branches={ | ||||
b"releases/0.0.1/hg-core-0.0.1.crate": SnapshotBranch( | b"releases/0.0.1/hg-core-0.0.1.crate": SnapshotBranch( | ||||
target=hash_to_bytes(expected_release_id), | target=hash_to_bytes(expected_release_id), | ||||
Show All 16 Lines | assert { | ||||
"release": 1, | "release": 1, | ||||
"revision": 0, | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( | assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( | ||||
name=b"0.0.1", | name=b"0.0.1", | ||||
message=b"Synthetic release for Crate source package hg-core version 0.0.1\n\n" | message=b"Synthetic release for Crate source package hg-core version 0.0.1\n", | ||||
b"Mercurial pure Rust core library, with no assumption " | |||||
b"on Python bindings (FFI)\n", | |||||
target=hash_to_bytes("674c3b0b54628d55b93a79dc7adf304efc01b371"), | target=hash_to_bytes("674c3b0b54628d55b93a79dc7adf304efc01b371"), | ||||
target_type=ObjectType.DIRECTORY, | target_type=ObjectType.DIRECTORY, | ||||
synthetic=True, | synthetic=True, | ||||
author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"), | author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"), | ||||
date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), | date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), | ||||
id=hash_to_bytes(expected_release_id), | id=hash_to_bytes(expected_release_id), | ||||
) | ) | ||||
def test_crates_loader_load_n_versions(datadir, requests_mock_datadir, swh_storage): | def test_crates_loader_load_n_versions( | ||||
url = CRATES_EXTRA[1]["url"] | datadir, requests_mock_datadir_visits, swh_storage, expected | ||||
): | |||||
url = expected[1]["url"] | |||||
loader = CratesLoader( | loader = CratesLoader( | ||||
swh_storage, | swh_storage, | ||||
url=CRATES_EXTRA[1]["url"], | url=url, | ||||
artifacts=CRATES_EXTRA[1]["artifacts"], | artifacts=expected[1]["artifacts"], | ||||
crates_metadata=expected[1]["crates_metadata"], | |||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
assert actual_load_status["status"] == "eventful" | assert actual_load_status["status"] == "eventful" | ||||
assert actual_load_status["snapshot_id"] is not None | assert actual_load_status["snapshot_id"] is not None | ||||
expected_snapshot_id = "3f8ca5908a570fa32270b07a0946bcffa88babd5" | expected_snapshot_id = "d53bf516948733e422eaceb95dd1ef317e6f5df9" | ||||
assert expected_snapshot_id == actual_load_status["snapshot_id"] | assert expected_snapshot_id == actual_load_status["snapshot_id"] | ||||
expected_snapshot = Snapshot( | expected_snapshot = Snapshot( | ||||
id=hash_to_bytes(expected_snapshot_id), | id=hash_to_bytes(expected_snapshot_id), | ||||
branches={ | branches={ | ||||
b"releases/0.4.0/micro-timer-0.4.0.crate": SnapshotBranch( | b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch( | ||||
target=hash_to_bytes("b038a927244c852fb3794aecbebdc70f68ddf067"), | target=hash_to_bytes("bd8d093b4ad56ca7e49aa0f709d945483b831915"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.3.1/micro-timer-0.3.1.crate": SnapshotBranch( | b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch( | ||||
target=hash_to_bytes("ea331a2ce755e6f0cd9d05c9be52accde68536c4"), | target=hash_to_bytes("a54bb00b38ae049ac4d7548a722bcd891811dbf6"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.3.0/micro-timer-0.3.0.crate": SnapshotBranch( | b"releases/0.1.2/micro-timer-0.1.2.crate": SnapshotBranch( | ||||
target=hash_to_bytes("7ea45f915ace083ed361bb12593625bf4cf1f5f2"), | target=hash_to_bytes("e234efbaef1999866d3b5c9a5676c42a5c4d1d3f"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.2.1/micro-timer-0.2.1.crate": SnapshotBranch( | b"releases/0.2.0/micro-timer-0.2.0.crate": SnapshotBranch( | ||||
target=hash_to_bytes("074f27605be8b759e5d7c638f026aac3709f58e5"), | target=hash_to_bytes("519bfdb898f91011eeb618d7d7aaa93097e57ab4"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.2.0/micro-timer-0.2.0.crate": SnapshotBranch( | b"releases/0.2.1/micro-timer-0.2.1.crate": SnapshotBranch( | ||||
target=hash_to_bytes("a1d642aaa54c5361f67e57adbd86e01f3a3276f8"), | target=hash_to_bytes("c1bdd5a5a0769c4fd10af92a43e66ce7a2394e8c"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.1.2/micro-timer-0.1.2.crate": SnapshotBranch( | b"releases/0.3.0/micro-timer-0.3.0.crate": SnapshotBranch( | ||||
target=hash_to_bytes("60f18ae067ce235bc60243bf5cdaaae474b11978"), | target=hash_to_bytes("d68b74713017c0b32a553f107d179be386bafdbc"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch( | b"releases/0.3.1/micro-timer-0.3.1.crate": SnapshotBranch( | ||||
target=hash_to_bytes("fd6c55dfd016d58647a2d44b29a3fd4e3afa7671"), | target=hash_to_bytes("ee11442a44d32562aedf731932ae4a8a9cf23feb"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch( | b"releases/0.4.0/micro-timer-0.4.0.crate": SnapshotBranch( | ||||
target=hash_to_bytes("3e07559a4b366a397b1ca154e72753ce27223ca1"), | target=hash_to_bytes("0018b25f87d0838f6bef3e94b3000043bc9c938d"), | ||||
target_type=TargetType.RELEASE, | target_type=TargetType.RELEASE, | ||||
), | ), | ||||
b"HEAD": SnapshotBranch( | b"HEAD": SnapshotBranch( | ||||
target=b"releases/0.4.0/micro-timer-0.4.0.crate", | target=b"releases/0.4.0/micro-timer-0.4.0.crate", | ||||
target_type=TargetType.ALIAS, | target_type=TargetType.ALIAS, | ||||
), | ), | ||||
}, | }, | ||||
) | ) | ||||
Show All 14 Lines | ): | ||||
assert_last_visit_matches( | assert_last_visit_matches( | ||||
swh_storage, | swh_storage, | ||||
url, | url, | ||||
status="full", | status="full", | ||||
type="crates", | type="crates", | ||||
snapshot=expected_snapshot.id, | snapshot=expected_snapshot.id, | ||||
) | ) | ||||
def test_crates_loader_load_multiple_visits_no_changes( | |||||
datadir, requests_mock_datadir_visits, requests_mock_datadir, swh_storage, expected | |||||
): | |||||
url = expected[0]["url"] | |||||
loader = CratesLoader( | |||||
swh_storage, | |||||
url=url, | |||||
artifacts=expected[0]["artifacts"], | |||||
crates_metadata=expected[0]["crates_metadata"], | |||||
) | |||||
visit_1_actual_load_status = loader.load() | |||||
assert visit_1_actual_load_status["status"] == "eventful" | |||||
assert visit_1_actual_load_status["snapshot_id"] is not None | |||||
expected_snapshot_id = "779fa94645a772fb9992c2bf80bf5e2ecfdb1b5c" | |||||
expected_release_id = "bb6f9b125867a8b4fa0b2febf890a317744e0140" | |||||
assert expected_snapshot_id == visit_1_actual_load_status["snapshot_id"] | |||||
expected_snapshot = Snapshot( | |||||
id=hash_to_bytes(visit_1_actual_load_status["snapshot_id"]), | |||||
branches={ | |||||
b"releases/0.0.1/hg-core-0.0.1.crate": SnapshotBranch( | |||||
target=hash_to_bytes(expected_release_id), | |||||
target_type=TargetType.RELEASE, | |||||
), | |||||
b"HEAD": SnapshotBranch( | |||||
target=b"releases/0.0.1/hg-core-0.0.1.crate", | |||||
target_type=TargetType.ALIAS, | |||||
), | |||||
}, | |||||
) | |||||
check_snapshot(expected_snapshot, swh_storage) | |||||
assert_last_visit_matches( | |||||
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot.id | |||||
) | |||||
stats = get_stats(swh_storage) | |||||
assert { | |||||
"content": 1, | |||||
"directory": 2, | |||||
"origin": 1, | |||||
"origin_visit": 1, | |||||
"release": 1, | |||||
"revision": 0, | |||||
"skipped_content": 0, | |||||
"snapshot": 1, | |||||
} == stats | |||||
assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( | |||||
name=b"0.0.1", | |||||
message=b"Synthetic release for Crate source package hg-core version 0.0.1\n", | |||||
target=hash_to_bytes("674c3b0b54628d55b93a79dc7adf304efc01b371"), | |||||
target_type=ObjectType.DIRECTORY, | |||||
synthetic=True, | |||||
author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"), | |||||
date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), | |||||
id=hash_to_bytes(expected_release_id), | |||||
) | |||||
loader_2 = CratesLoader( | |||||
swh_storage, | |||||
url=url, | |||||
artifacts=expected[0]["artifacts"], | |||||
crates_metadata=expected[0]["crates_metadata"], | |||||
) | |||||
actual_load_status2 = loader_2.load() | |||||
assert actual_load_status2 == { | |||||
"status": "uneventful", | |||||
"snapshot_id": actual_load_status2["snapshot_id"], | |||||
} | |||||
visit_status2 = assert_last_visit_matches( | |||||
swh_storage, url, status="full", type="crates" | |||||
) | |||||
stats2 = get_stats(swh_storage) | |||||
expected_stats2 = stats.copy() | |||||
expected_stats2["origin_visit"] = 1 + 1 | |||||
assert expected_stats2 == stats2 | |||||
# same snapshot | |||||
assert visit_status2.snapshot == expected_snapshot.id | |||||
def test_crates_loader_load_multiple_version_incremental( | |||||
datadir, requests_mock_datadir_visits, swh_storage, expected | |||||
): | |||||
url = expected[1]["url"] | |||||
# one version in artifacts | |||||
artifacts_0 = [ | |||||
artifact | |||||
for artifact in expected[1]["artifacts"] | |||||
if artifact["version"] in ["0.1.0"] | |||||
] | |||||
crates_metadata_0 = [ | |||||
artifact | |||||
for artifact in expected[1]["crates_metadata"] | |||||
if artifact["version"] in ["0.1.0"] | |||||
] | |||||
# two versions in artifacts | |||||
artifacts_1 = [ | |||||
artifact | |||||
for artifact in expected[1]["artifacts"] | |||||
if artifact["version"] in ["0.1.0", "0.1.1"] | |||||
] | |||||
crates_metadata_1 = [ | |||||
artifact | |||||
for artifact in expected[1]["crates_metadata"] | |||||
if artifact["version"] in ["0.1.0", "0.1.1"] | |||||
] | |||||
# Visit 1 | |||||
loader = CratesLoader( | |||||
swh_storage, | |||||
url=url, | |||||
artifacts=artifacts_0, | |||||
crates_metadata=crates_metadata_0, | |||||
) | |||||
visit1_actual_load_status = loader.load() | |||||
visit1_stats = get_stats(swh_storage) | |||||
expected_snapshot_id = hash_to_bytes("729d99ab2e68cde99c425251ac5725c478d686df") | |||||
assert visit1_actual_load_status == { | |||||
"status": "eventful", | |||||
"snapshot_id": expected_snapshot_id.hex(), | |||||
vlorentz: Create a new loader instance, so tests behave more like production | |||||
} | |||||
assert_last_visit_matches( | |||||
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot_id | |||||
) | |||||
assert { | |||||
"content": 1, | |||||
"directory": 2, | |||||
"origin": 1, | |||||
"origin_visit": 1, | |||||
"release": 1, | |||||
"revision": 0, | |||||
"skipped_content": 0, | |||||
"snapshot": 1, | |||||
} == visit1_stats | |||||
# Visit 2 | |||||
loader = CratesLoader( | |||||
swh_storage, | |||||
url=url, | |||||
artifacts=artifacts_1, | |||||
crates_metadata=crates_metadata_1, | |||||
) | |||||
visit2_actual_load_status = loader.load() | |||||
visit2_stats = get_stats(swh_storage) | |||||
assert visit2_actual_load_status["status"] == "eventful", visit2_actual_load_status | |||||
expected_snapshot_id2 = hash_to_bytes("eff31762ab65175192d6e1f0570d8b2e4176ff0d") | |||||
assert visit2_actual_load_status == { | |||||
"status": "eventful", | |||||
"snapshot_id": expected_snapshot_id2.hex(), | |||||
} | |||||
assert_last_visit_matches( | |||||
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot_id2 | |||||
) | |||||
expected_snapshot = Snapshot( | |||||
id=expected_snapshot_id2, | |||||
branches={ | |||||
b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch( | |||||
target=hash_to_bytes("a54bb00b38ae049ac4d7548a722bcd891811dbf6"), | |||||
target_type=TargetType.RELEASE, | |||||
), | |||||
b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch( | |||||
target=hash_to_bytes("bd8d093b4ad56ca7e49aa0f709d945483b831915"), | |||||
target_type=TargetType.RELEASE, | |||||
), | |||||
b"HEAD": SnapshotBranch( | |||||
target=b"releases/0.1.1/micro-timer-0.1.1.crate", | |||||
target_type=TargetType.ALIAS, | |||||
), | |||||
}, | |||||
) | |||||
assert_last_visit_matches( | |||||
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot.id | |||||
) | |||||
check_snapshot(expected_snapshot, swh_storage) | |||||
assert { | |||||
"content": 1 + 1, | |||||
"directory": 2 + 2, | |||||
"origin": 1, | |||||
"origin_visit": 1 + 1, | |||||
"release": 1 + 1, | |||||
"revision": 0, | |||||
"skipped_content": 0, | |||||
"snapshot": 1 + 1, | |||||
} == visit2_stats | |||||
def test_crates_loader_raw_extrinsic_metadata( | |||||
datadir, requests_mock_datadir, swh_storage, expected | |||||
): | |||||
loader = CratesLoader( | |||||
swh_storage, | |||||
url=expected[0]["url"], | |||||
artifacts=expected[0]["artifacts"], | |||||
crates_metadata=expected[0]["crates_metadata"], | |||||
) | |||||
actual_load_status = loader.load() | |||||
assert actual_load_status["status"] == "eventful" | |||||
assert actual_load_status["snapshot_id"] is not None | |||||
expected_release_id = "bb6f9b125867a8b4fa0b2febf890a317744e0140" | |||||
release = swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] | |||||
release_swhid = CoreSWHID( | |||||
object_type=OType.RELEASE, object_id=hash_to_bytes(expected_release_id) | |||||
) | |||||
directory_swhid = ExtendedSWHID( | |||||
object_type=ExtendedObjectType.DIRECTORY, object_id=release.target | |||||
) | |||||
metadata_authority = MetadataAuthority( | |||||
type=MetadataAuthorityType.FORGE, | |||||
url="https://crates.io/", | |||||
) | |||||
expected_metadata = [ | |||||
RawExtrinsicMetadata( | |||||
target=directory_swhid, | |||||
authority=metadata_authority, | |||||
fetcher=MetadataFetcher( | |||||
name="swh.loader.package.crates.loader.CratesLoader", | |||||
version=__version__, | |||||
), | |||||
discovery_date=loader.visit_date, | |||||
format="crates-package-json", | |||||
metadata=json.dumps( | |||||
[ | |||||
{ | |||||
"version": "0.0.1", | |||||
"yanked": False, | |||||
"last_update": "2019-04-16T18:48:11.404457+00:00", | |||||
} | |||||
] | |||||
).encode(), | |||||
origin=expected[0]["url"], | |||||
release=release_swhid, | |||||
), | |||||
] | |||||
assert swh_storage.raw_extrinsic_metadata_get( | |||||
directory_swhid, | |||||
metadata_authority, | |||||
) == PagedResult( | |||||
next_page_token=None, | |||||
results=expected_metadata, | |||||
) |
Create a new loader instance, so tests behave more like production