Differential D5423 Diff 19380 swh/loader/package/deposit/tests/test_deposit.py

Changeset View

Standalone View

swh/loader/package/deposit/tests/test_deposit.py

# Copyright (C) 2019-2021 The Software Heritage developers		# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import json		import json
import re		import re
from typing import List		from typing import List

import pytest		import pytest

from swh.core.pytest_plugin import requests_mock_datadir_factory		from swh.core.pytest_plugin import requests_mock_datadir_factory
from swh.loader.package.deposit.loader import ApiClient, DepositLoader		from swh.loader.package.deposit.loader import ApiClient, DepositLoader
from swh.loader.package.loader import now		from swh.loader.package.loader import now
from swh.loader.package.tests.common import check_metadata_paths
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats		from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats
from swh.model.hashutil import hash_to_bytes, hash_to_hex		from swh.model.hashutil import hash_to_bytes, hash_to_hex
from swh.model.identifiers import (		from swh.model.identifiers import (
CoreSWHID,		CoreSWHID,
ExtendedObjectType,		ExtendedObjectType,
ExtendedSWHID,		ExtendedSWHID,
ObjectType,		ObjectType,
)		)
▲ Show 20 Lines • Show All 129 Lines • ▼ Show 20 Lines	):
body = update_query.json()		body = update_query.json()
expected_body = {		expected_body = {
"status": "failed",		"status": "failed",
}		}

assert body == expected_body		assert body == expected_body


def test_deposit_revision_metadata_structure(
swh_storage, deposit_client, requests_mock_datadir
):
url = "https://hal-test.archives-ouvertes.fr/some-external-id"
deposit_id = 666
loader = DepositLoader(
swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip"
)

actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
expected_revision_id = hash_to_bytes("637318680351f5d78856d13264faebbd91efe9bb")
revision = loader.storage.revision_get([expected_revision_id])[0]
assert revision is not None

check_metadata_paths(
revision.metadata,
paths=[
("extrinsic.provider", str),
("extrinsic.when", str),
("extrinsic.raw", dict),
("original_artifact", list),
],
)

# Only 2 top-level keys now
assert set(revision.metadata.keys()) == {"extrinsic", "original_artifact"}

for original_artifact in revision.metadata["original_artifact"]:
check_metadata_paths(
original_artifact,
paths=[("filename", str), ("length", int), ("checksums", dict),],
)


def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir):		def test_deposit_loading_ok(swh_storage, deposit_client, requests_mock_datadir):
url = "https://hal-test.archives-ouvertes.fr/some-external-id"		url = "https://hal-test.archives-ouvertes.fr/some-external-id"
deposit_id = 666		deposit_id = 666
loader = DepositLoader(		loader = DepositLoader(
swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip"		swh_storage, url, deposit_id, deposit_client, default_filename="archive.zip"
)		)

actual_load_status = loader.load()		actual_load_status = loader.load()
▲ Show 20 Lines • Show All 126 Lines • ▼ Show 20 Lines	def test_deposit_loading_ok_2(swh_storage, deposit_client, requests_mock_datadir):
raw_meta = loader.client.metadata_get(deposit_id)		raw_meta = loader.client.metadata_get(deposit_id)
# Ensure the date fields are set appropriately in the revision		# Ensure the date fields are set appropriately in the revision

# Retrieve the revision		# Retrieve the revision
revision = loader.storage.revision_get([hash_to_bytes(revision_id)])[0]		revision = loader.storage.revision_get([hash_to_bytes(revision_id)])[0]
assert revision		assert revision
assert revision.date.to_dict() == raw_meta["deposit"]["author_date"]		assert revision.date.to_dict() == raw_meta["deposit"]["author_date"]
assert revision.committer_date.to_dict() == raw_meta["deposit"]["committer_date"]		assert revision.committer_date.to_dict() == raw_meta["deposit"]["committer_date"]
		assert not revision.metadata
read_api = f"{DEPOSIT_URL}/{deposit_id}/meta/"

provider = {		provider = {
"provider_name": "hal",		"provider_name": "hal",
"provider_type": "deposit_client",		"provider_type": "deposit_client",
"provider_url": "https://hal-test.archives-ouvertes.fr/",		"provider_url": "https://hal-test.archives-ouvertes.fr/",
"metadata": None,		"metadata": None,
}		}
tool = {		tool = {
"name": "swh-deposit",		"name": "swh-deposit",
"version": "0.0.1",		"version": "0.0.1",
"configuration": {"sword_version": "2"},		"configuration": {"sword_version": "2"},
}		}
assert revision.metadata == {
"extrinsic": {
"provider": read_api,
"raw": {
"origin": {"type": "deposit", "url": url,},
"origin_metadata": {
"metadata": raw_meta["metadata_dict"],
"provider": provider,
"tool": tool,
},
},
"when": revision.metadata["extrinsic"]["when"], # dynamic
},
"original_artifact": [
{
"checksums": {
"sha1": "f8c63d7c890a7453498e6cf9fef215d85ec6801d",
"sha256": "474bf646aeeff6d945eb752b1a9f8a40f3d81a88909ee7bd2d08cc822aa361e6", # noqa
},
"filename": "archive.zip",
"length": 956830,
"url": "https://deposit.softwareheritage.org/1/private/777/raw/",
}
],
}

fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",)		fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",)

authority = MetadataAuthority(		authority = MetadataAuthority(
type=MetadataAuthorityType.DEPOSIT_CLIENT,		type=MetadataAuthorityType.DEPOSIT_CLIENT,
url="https://hal-test.archives-ouvertes.fr/",		url="https://hal-test.archives-ouvertes.fr/",
)		)

▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines