Page MenuHomeSoftware Heritage
Differential D8171 Diff 29506 swh/loader/package/crates/tests/test_crates.py

Changeset View

Standalone View

View Options

swh/loader/package/crates/tests/test_crates.py

# Copyright (C) 2022 The Software Heritage developers # Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import pytest import pytest
import requests
from swh.loader.package.crates.loader import CratesLoader from swh.loader.package.crates.loader import CratesLoader
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats
from swh.model.hashutil import hash_to_bytes from swh.model.hashutil import hash_to_bytes
from swh.model.model import ( from swh.model.model import (
ObjectType, ObjectType,
Person, Person,
Release, Release,
▲ Show 20 Lines • Show All 188 Lines • ▼ Show 20 Lines assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release(
target_type=ObjectType.DIRECTORY, target_type=ObjectType.DIRECTORY,
synthetic=True, synthetic=True,
author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"), author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"),
date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"),
id=hash_to_bytes(expected_release_id), id=hash_to_bytes(expected_release_id),
) )
def test_crates_loader_load_n_versions(datadir, requests_mock_datadir, swh_storage): def test_crates_loader_load_n_versions(
datadir, requests_mock_datadir_visits, swh_storage
):
url = CRATES_EXTRA[1]["url"] url = CRATES_EXTRA[1]["url"]
# Force visits to get to visit_2
requests.get(url)
requests.get(url)
loader = CratesLoader( loader = CratesLoader(
swh_storage, swh_storage,
url=CRATES_EXTRA[1]["url"], url=url,
artifacts=CRATES_EXTRA[1]["artifacts"], artifacts=CRATES_EXTRA[1]["artifacts"],
) )
actual_load_status = loader.load() actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful" assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None assert actual_load_status["snapshot_id"] is not None
expected_snapshot_id = "3f8ca5908a570fa32270b07a0946bcffa88babd5" expected_snapshot_id = "3f8ca5908a570fa32270b07a0946bcffa88babd5"
assert expected_snapshot_id == actual_load_status["snapshot_id"] assert expected_snapshot_id == actual_load_status["snapshot_id"]
expected_snapshot = Snapshot( expected_snapshot = Snapshot(
id=hash_to_bytes(expected_snapshot_id), id=hash_to_bytes(expected_snapshot_id),
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines):
assert_last_visit_matches( assert_last_visit_matches(
swh_storage, swh_storage,
url, url,
status="full", status="full",
type="crates", type="crates",
snapshot=expected_snapshot.id, snapshot=expected_snapshot.id,
) )
def test_crates_loader_load_multiple_visits_no_changes(
datadir, requests_mock_datadir_visits, requests_mock_datadir, swh_storage
):
url = CRATES_EXTRA[0]["url"]
loader = CratesLoader(
swh_storage,
url=url,
artifacts=CRATES_EXTRA[0]["artifacts"],
)
visit_1_actual_load_status = loader.load()
assert visit_1_actual_load_status["status"] == "eventful"
assert visit_1_actual_load_status["snapshot_id"] is not None
expected_snapshot_id = "b3affb4949eb89b244f0e1d1fe235fc1d26bde76"
expected_release_id = "237c4cdd44a90e620795e5a07ebcc72bc82487f7"
assert expected_snapshot_id == visit_1_actual_load_status["snapshot_id"]
expected_snapshot = Snapshot(
id=hash_to_bytes(visit_1_actual_load_status["snapshot_id"]),
branches={
b"releases/0.0.1/hg-core-0.0.1.crate": SnapshotBranch(
target=hash_to_bytes(expected_release_id),
target_type=TargetType.RELEASE,
),
b"HEAD": SnapshotBranch(
target=b"releases/0.0.1/hg-core-0.0.1.crate",
target_type=TargetType.ALIAS,
),
},
)
check_snapshot(expected_snapshot, swh_storage)
assert_last_visit_matches(
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot.id
)
stats = get_stats(swh_storage)
assert {
"content": 1,
"directory": 2,
"origin": 1,
"origin_visit": 1,
"release": 1,
"revision": 0,
"skipped_content": 0,
"snapshot": 1,
} == stats
assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release(
name=b"0.0.1",
message=b"Synthetic release for Crate source package hg-core version 0.0.1\n\n"
b"Mercurial pure Rust core library, with no assumption "
b"on Python bindings (FFI)\n",
target=hash_to_bytes("674c3b0b54628d55b93a79dc7adf304efc01b371"),
target_type=ObjectType.DIRECTORY,
synthetic=True,
author=Person.from_fullname(b"Georges Racinet <georges.racinet@octobus.net>"),
date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"),
id=hash_to_bytes(expected_release_id),
)
actual_load_status2 = loader.load()
assert actual_load_status2 == {
"status": "uneventful",
"snapshot_id": actual_load_status2["snapshot_id"],
}
visit_status2 = assert_last_visit_matches(
swh_storage, url, status="full", type="crates"
)
stats2 = get_stats(swh_storage)
expected_stats2 = stats.copy()
expected_stats2["origin_visit"] = 1 + 1
assert expected_stats2 == stats2
# same snapshot
assert visit_status2.snapshot == expected_snapshot.id
def test_crates_loader_load_multiple_version_incremental(
datadir, requests_mock_datadir_visits, swh_storage
):
url = CRATES_EXTRA[1]["url"]
# one version in artifacts
artifacts_0 = CRATES_EXTRA[1]["artifacts"].copy()
artifacts_0 = [
version for version in artifacts_0 if version["version"] in ["0.1.0"]
]
# two versions in artifacts
artifacts_1 = CRATES_EXTRA[1]["artifacts"].copy()
artifacts_1 = [
version for version in artifacts_1 if version["version"] in ["0.1.0", "0.1.1"]
]
# Visit 1
loader = CratesLoader(
swh_storage,
url=url,
artifacts=artifacts_0,
)
visit1_actual_load_status = loader.load()
visit1_stats = get_stats(swh_storage)
expected_snapshot_id = hash_to_bytes("42c66a04aa69b78ef5009fb3dfc2bbd0e1dd4c34")
assert visit1_actual_load_status == {
"status": "eventful",
"snapshot_id": expected_snapshot_id.hex(),
}
assert_last_visit_matches(
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot_id
)
assert {
"content": 1,
"directory": 2,
"origin": 1,
"origin_visit": 1,
"release": 1,
"revision": 0,
"skipped_content": 0,
"snapshot": 1,
} == visit1_stats
# Visit 2
# Reset internal state
del loader._cached__raw_info
del loader._cached_info
vlorentzUnsubmitted
Done
Inline Actions

Create a new loader instance, so tests behave more like production

vlorentz: Create a new loader instance, so tests behave more like production
# set loader artifacts to list two versions
loader.artifacts = {artifact["version"]: artifact for artifact in artifacts_1}
visit2_actual_load_status = loader.load()
visit2_stats = get_stats(swh_storage)
assert visit2_actual_load_status["status"] == "eventful", visit2_actual_load_status
expected_snapshot_id2 = hash_to_bytes("2dff2ce07d446f0cd16d052ca969547eff5a2be5")
assert visit2_actual_load_status == {
"status": "eventful",
"snapshot_id": expected_snapshot_id2.hex(),
}
assert_last_visit_matches(
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot_id2
)
expected_snapshot = Snapshot(
id=expected_snapshot_id2,
branches={
b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch(
target=hash_to_bytes("fd6c55dfd016d58647a2d44b29a3fd4e3afa7671"),
target_type=TargetType.RELEASE,
),
b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch(
target=hash_to_bytes("3e07559a4b366a397b1ca154e72753ce27223ca1"),
target_type=TargetType.RELEASE,
),
b"HEAD": SnapshotBranch(
target=b"releases/0.1.1/micro-timer-0.1.1.crate",
target_type=TargetType.ALIAS,
),
},
)
assert_last_visit_matches(
swh_storage, url, status="full", type="crates", snapshot=expected_snapshot.id
)
check_snapshot(expected_snapshot, swh_storage)
assert {
"content": 1 + 1,
"directory": 2 + 2,
"origin": 1,
"origin_visit": 1 + 1,
"release": 1 + 1,
"revision": 0,
"skipped_content": 0,
"snapshot": 1 + 1,
} == visit2_stats
About · Developer information · Legal