Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/deposit/tests/test_deposit.py
# Copyright (C) 2019-2021 The Software Heritage developers | # Copyright (C) 2019-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import json | import json | ||||
import re | import re | ||||
from typing import List | from typing import List | ||||
import pytest | import pytest | ||||
from swh.core.pytest_plugin import requests_mock_datadir_factory | from swh.core.pytest_plugin import requests_mock_datadir_factory | ||||
from swh.loader.package.deposit.loader import ApiClient, DepositLoader | from swh.loader.package.deposit.loader import ApiClient, DepositLoader | ||||
from swh.loader.package.loader import now | from swh.loader.package.loader import now | ||||
from swh.loader.package.tests.common import check_metadata_paths | |||||
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats | ||||
from swh.model.hashutil import hash_to_bytes, hash_to_hex | from swh.model.hashutil import hash_to_bytes, hash_to_hex | ||||
from swh.model.identifiers import ( | from swh.model.identifiers import ( | ||||
CoreSWHID, | CoreSWHID, | ||||
ExtendedObjectType, | ExtendedObjectType, | ||||
ExtendedSWHID, | ExtendedSWHID, | ||||
ObjectType, | ObjectType, | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines | ): | ||||
body = update_query.json() | body = update_query.json() | ||||
expected_body = { | expected_body = { | ||||
"status": "failed", | "status": "failed", | ||||
} | } | ||||
assert body == expected_body | assert body == expected_body | ||||
def test_deposit_revision_metadata_structure( | |||||
swh_storage, deposit_client, requests_mock_datadir | |||||
): | |||||
url = "https://hal-test.archives-ouvertes.fr/some-external-id" | |||||
deposit_id = 666 | |||||
loader = DepositLoader( | |||||
swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | |||||
) | |||||
actual_load_status = loader.load() | |||||
assert actual_load_status["status"] == "eventful" | |||||
assert actual_load_status["snapshot_id"] is not None | |||||
expected_revision_id = hash_to_bytes("637318680351f5d78856d13264faebbd91efe9bb") | |||||
revision = loader.storage.revision_get([expected_revision_id])[0] | |||||
assert revision is not None | |||||
check_metadata_paths( | |||||
revision.metadata, | |||||
paths=[ | |||||
("extrinsic.provider", str), | |||||
("extrinsic.when", str), | |||||
("extrinsic.raw", dict), | |||||
("original_artifact", list), | |||||
], | |||||
) | |||||
# Only 2 top-level keys now | |||||
assert set(revision.metadata.keys()) == {"extrinsic", "original_artifact"} | |||||
for original_artifact in revision.metadata["original_artifact"]: | |||||
check_metadata_paths( | |||||
original_artifact, | |||||
paths=[("filename", str), ("length", int), ("checksums", dict),], | |||||
) | |||||
def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir): | def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir): | ||||
url = "https://hal-test.archives-ouvertes.fr/some-external-id" | url = "https://hal-test.archives-ouvertes.fr/some-external-id" | ||||
deposit_id = 666 | deposit_id = 666 | ||||
loader = DepositLoader( | loader = DepositLoader( | ||||
swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip" | ||||
) | ) | ||||
actual_load_status = loader.load() | actual_load_status = loader.load() | ||||
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines | def test_deposit_loading_ok_2(swh_storage, deposit_client, requests_mock_datadir): | ||||
raw_meta = loader.client.metadata_get(deposit_id) | raw_meta = loader.client.metadata_get(deposit_id) | ||||
# Ensure the date fields are set appropriately in the revision | # Ensure the date fields are set appropriately in the revision | ||||
# Retrieve the revision | # Retrieve the revision | ||||
revision = loader.storage.revision_get([hash_to_bytes(revision_id)])[0] | revision = loader.storage.revision_get([hash_to_bytes(revision_id)])[0] | ||||
assert revision | assert revision | ||||
assert revision.date.to_dict() == raw_meta["deposit"]["author_date"] | assert revision.date.to_dict() == raw_meta["deposit"]["author_date"] | ||||
assert revision.committer_date.to_dict() == raw_meta["deposit"]["committer_date"] | assert revision.committer_date.to_dict() == raw_meta["deposit"]["committer_date"] | ||||
assert not revision.metadata | |||||
read_api = f"{DEPOSIT_URL}/{deposit_id}/meta/" | |||||
provider = { | provider = { | ||||
"provider_name": "hal", | "provider_name": "hal", | ||||
"provider_type": "deposit_client", | "provider_type": "deposit_client", | ||||
"provider_url": "https://hal-test.archives-ouvertes.fr/", | "provider_url": "https://hal-test.archives-ouvertes.fr/", | ||||
"metadata": None, | "metadata": None, | ||||
} | } | ||||
tool = { | tool = { | ||||
"name": "swh-deposit", | "name": "swh-deposit", | ||||
"version": "0.0.1", | "version": "0.0.1", | ||||
"configuration": {"sword_version": "2"}, | "configuration": {"sword_version": "2"}, | ||||
} | } | ||||
assert revision.metadata == { | |||||
"extrinsic": { | |||||
"provider": read_api, | |||||
"raw": { | |||||
"origin": {"type": "deposit", "url": url,}, | |||||
"origin_metadata": { | |||||
"metadata": raw_meta["metadata_dict"], | |||||
"provider": provider, | |||||
"tool": tool, | |||||
}, | |||||
}, | |||||
"when": revision.metadata["extrinsic"]["when"], # dynamic | |||||
}, | |||||
"original_artifact": [ | |||||
{ | |||||
"checksums": { | |||||
"sha1": "f8c63d7c890a7453498e6cf9fef215d85ec6801d", | |||||
"sha256": "474bf646aeeff6d945eb752b1a9f8a40f3d81a88909ee7bd2d08cc822aa361e6", # noqa | |||||
}, | |||||
"filename": "archive.zip", | |||||
"length": 956830, | |||||
"url": "https://deposit.softwareheritage.org/1/private/777/raw/", | |||||
} | |||||
], | |||||
} | |||||
fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",) | fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",) | ||||
authority = MetadataAuthority( | authority = MetadataAuthority( | ||||
type=MetadataAuthorityType.DEPOSIT_CLIENT, | type=MetadataAuthorityType.DEPOSIT_CLIENT, | ||||
url="https://hal-test.archives-ouvertes.fr/", | url="https://hal-test.archives-ouvertes.fr/", | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines |