Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/deposit/tests/test_deposit.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import re | import re | ||||
import pytest | import pytest | ||||
from swh.model.hashutil import hash_to_bytes | from swh.model.hashutil import hash_to_bytes, hash_to_hex | ||||
from swh.loader.package.deposit.loader import DepositLoader | from swh.loader.package.deposit.loader import DepositLoader | ||||
from swh.loader.package.tests.common import ( | from swh.loader.package.tests.common import ( | ||||
check_snapshot, | check_snapshot, | ||||
check_metadata_paths, | check_metadata_paths, | ||||
get_stats, | get_stats, | ||||
) | ) | ||||
from swh.core.pytest_plugin import requests_mock_datadir_factory | from swh.core.pytest_plugin import requests_mock_datadir_factory | ||||
DEPOSIT_URL = "https://deposit.softwareheritage.org/1/private" | |||||
@pytest.fixture | @pytest.fixture | ||||
def requests_mock_datadir(requests_mock_datadir): | def requests_mock_datadir(requests_mock_datadir): | ||||
"""Enhance default mock data to mock put requests as the loader does some | """Enhance default mock data to mock put requests as the loader does some | ||||
internal update queries there. | internal update queries there. | ||||
""" | """ | ||||
requests_mock_datadir.put(re.compile("https")) | requests_mock_datadir.put(re.compile("https")) | ||||
return requests_mock_datadir | return requests_mock_datadir | ||||
Show All 33 Lines | assert { | ||||
"release": 0, | "release": 0, | ||||
"revision": 0, | "revision": 0, | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 0, | "snapshot": 0, | ||||
} == stats | } == stats | ||||
requests_mock_datadir_missing_one = requests_mock_datadir_factory( | requests_mock_datadir_missing_one = requests_mock_datadir_factory( | ||||
ignore_urls=["https://deposit.softwareheritage.org/1/private/666/raw/",] | ignore_urls=[f"{DEPOSIT_URL}/666/raw/",] | ||||
) | ) | ||||
def test_deposit_loading_failure_to_retrieve_1_artifact( | def test_deposit_loading_failure_to_retrieve_1_artifact( | ||||
swh_config, requests_mock_datadir_missing_one | swh_config, requests_mock_datadir_missing_one | ||||
): | ): | ||||
"""Deposit with missing artifact ends up with an uneventful/partial visit | """Deposit with missing artifact ends up with an uneventful/partial visit | ||||
▲ Show 20 Lines • Show All 82 Lines • ▼ Show 20 Lines | assert { | ||||
"skipped_content": 0, | "skipped_content": 0, | ||||
"snapshot": 1, | "snapshot": 1, | ||||
} == stats | } == stats | ||||
origin_visit = loader.storage.origin_visit_get_latest(url) | origin_visit = loader.storage.origin_visit_get_latest(url) | ||||
assert origin_visit["status"] == "full" | assert origin_visit["status"] == "full" | ||||
assert origin_visit["type"] == "deposit" | assert origin_visit["type"] == "deposit" | ||||
revision_id = "637318680351f5d78856d13264faebbd91efe9bb" | |||||
expected_branches = { | expected_branches = { | ||||
"HEAD": { | "HEAD": {"target": revision_id, "target_type": "revision",}, | ||||
"target": "637318680351f5d78856d13264faebbd91efe9bb", | |||||
"target_type": "revision", | |||||
}, | |||||
} | } | ||||
expected_snapshot = { | expected_snapshot = { | ||||
"id": expected_snapshot_id, | "id": expected_snapshot_id, | ||||
"branches": expected_branches, | "branches": expected_branches, | ||||
} | } | ||||
check_snapshot(expected_snapshot, storage=loader.storage) | check_snapshot(expected_snapshot, storage=loader.storage) | ||||
revision = next(loader.storage.revision_get([hash_to_bytes(revision_id)])) | |||||
assert revision | |||||
# check metadata | # check metadata | ||||
tool = { | tool = { | ||||
"name": "swh-deposit", | "name": "swh-deposit", | ||||
"version": "0.0.1", | "version": "0.0.1", | ||||
"configuration": {"sword_version": "2",}, | "configuration": {"sword_version": "2",}, | ||||
} | } | ||||
Show All 19 Lines | def test_deposit_loading_ok(swh_config, requests_mock_datadir): | ||||
assert isinstance(metadata, list) | assert isinstance(metadata, list) | ||||
assert len(metadata) == 1 | assert len(metadata) == 1 | ||||
metadata0 = metadata[0] | metadata0 = metadata[0] | ||||
assert metadata0["provider_id"] == provider["id"] | assert metadata0["provider_id"] == provider["id"] | ||||
assert metadata0["provider_type"] == "deposit_client" | assert metadata0["provider_type"] == "deposit_client" | ||||
assert metadata0["tool_id"] == tool["id"] | assert metadata0["tool_id"] == tool["id"] | ||||
# Retrieve the information for deposit status update query to the deposit | |||||
urls = [ | |||||
m | |||||
for m in requests_mock_datadir.request_history | |||||
if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | |||||
] | |||||
assert len(urls) == 1 | |||||
update_query = urls[0] | |||||
body = update_query.json() | |||||
expected_body = { | |||||
"status": "done", | |||||
"revision_id": revision_id, | |||||
"directory_id": hash_to_hex(revision["directory"]), | |||||
"snapshot_id": expected_snapshot_id, | |||||
"origin_url": url, | |||||
} | |||||
assert body == expected_body | |||||
def test_deposit_loading_ok_2(swh_config, requests_mock_datadir): | def test_deposit_loading_ok_2(swh_config, requests_mock_datadir): | ||||
"""Field dates should be se appropriately | """Field dates should be se appropriately | ||||
""" | """ | ||||
external_id = "some-external-id" | external_id = "some-external-id" | ||||
url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | url = f"https://hal-test.archives-ouvertes.fr/{external_id}" | ||||
deposit_id = 777 | deposit_id = 777 | ||||
Show All 26 Lines | def test_deposit_loading_ok_2(swh_config, requests_mock_datadir): | ||||
# Ensure the date fields are set appropriately in the revision | # Ensure the date fields are set appropriately in the revision | ||||
# Retrieve the revision | # Retrieve the revision | ||||
revision = next(loader.storage.revision_get([hash_to_bytes(revision_id)])) | revision = next(loader.storage.revision_get([hash_to_bytes(revision_id)])) | ||||
assert revision | assert revision | ||||
assert revision["date"] == raw_meta["deposit"]["author_date"] | assert revision["date"] == raw_meta["deposit"]["author_date"] | ||||
assert revision["committer_date"] == raw_meta["deposit"]["committer_date"] | assert revision["committer_date"] == raw_meta["deposit"]["committer_date"] | ||||
read_api = f"https://deposit.softwareheritage.org/1/private/{deposit_id}/meta/" | read_api = f"{DEPOSIT_URL}/{deposit_id}/meta/" | ||||
assert revision["metadata"] == { | assert revision["metadata"] == { | ||||
"extrinsic": { | "extrinsic": { | ||||
"provider": read_api, | "provider": read_api, | ||||
"raw": { | "raw": { | ||||
"origin": {"type": "deposit", "url": url,}, | "origin": {"type": "deposit", "url": url,}, | ||||
"origin_metadata": { | "origin_metadata": { | ||||
"metadata": { | "metadata": { | ||||
▲ Show 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | assert origin_meta == { | ||||
}, | }, | ||||
"origin_url": "https://hal-test.archives-ouvertes.fr/some-external-id", | "origin_url": "https://hal-test.archives-ouvertes.fr/some-external-id", | ||||
"provider_id": 1, | "provider_id": 1, | ||||
"provider_name": "hal", | "provider_name": "hal", | ||||
"provider_type": "deposit_client", | "provider_type": "deposit_client", | ||||
"provider_url": "https://hal-test.archives-ouvertes.fr/", | "provider_url": "https://hal-test.archives-ouvertes.fr/", | ||||
"tool_id": 1, | "tool_id": 1, | ||||
} | } | ||||
# Retrieve the information for deposit status update query to the deposit | |||||
urls = [ | |||||
m | |||||
for m in requests_mock_datadir.request_history | |||||
if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | |||||
] | |||||
assert len(urls) == 1 | |||||
update_query = urls[0] | |||||
body = update_query.json() | |||||
expected_body = { | |||||
"status": "done", | |||||
"revision_id": revision_id, | |||||
"directory_id": hash_to_hex(revision["directory"]), | |||||
"snapshot_id": expected_snapshot_id, | |||||
"origin_url": url, | |||||
} | |||||
assert body == expected_body |