Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/deposit/tests/test_deposit.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import re | import re | ||||
| import pytest | import pytest | ||||
| from swh.model.hashutil import hash_to_bytes, hash_to_hex | from swh.model.hashutil import hash_to_bytes, hash_to_hex | ||||
| from swh.model.model import Snapshot, SnapshotBranch, TargetType | from swh.model.model import ( | ||||
| Snapshot, | |||||
| SnapshotBranch, | |||||
| TargetType, | |||||
| MetadataAuthority, | |||||
| MetadataAuthorityType, | |||||
| MetadataFetcher, | |||||
| MetadataTargetType, | |||||
| RawExtrinsicMetadata, | |||||
| ) | |||||
| from swh.loader.package.deposit.loader import DepositLoader | from swh.loader.package.deposit.loader import DepositLoader | ||||
| from swh.loader.package.tests.common import check_metadata_paths | from swh.loader.package.tests.common import check_metadata_paths | ||||
| from swh.loader.tests import ( | from swh.loader.tests import ( | ||||
| assert_last_visit_matches, | assert_last_visit_matches, | ||||
| check_snapshot, | check_snapshot, | ||||
| get_stats, | get_stats, | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 161 Lines • ▼ Show 20 Lines | def test_deposit_loading_ok(swh_config, requests_mock_datadir): | ||||
| ) | ) | ||||
| check_snapshot(expected_snapshot, storage=loader.storage) | check_snapshot(expected_snapshot, storage=loader.storage) | ||||
| revision = next(loader.storage.revision_get([revision_id])) | revision = next(loader.storage.revision_get([revision_id])) | ||||
| assert revision | assert revision | ||||
| # check metadata | # check metadata | ||||
| fetcher = { | fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",) | ||||
| "name": "swh-deposit", | |||||
| "version": "0.0.1", | |||||
| } | |||||
| authority = { | authority = MetadataAuthority( | ||||
| "type": "deposit_client", | type=MetadataAuthorityType.DEPOSIT, | ||||
| "url": "https://hal-test.archives-ouvertes.fr/", | url="https://hal-test.archives-ouvertes.fr/", | ||||
| } | ) | ||||
| orig_meta = loader.storage.origin_metadata_get(url, authority) | orig_meta = loader.storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, url, authority | |||||
| ) | |||||
| assert orig_meta is not None | assert orig_meta is not None | ||||
| assert isinstance(orig_meta, dict) | assert isinstance(orig_meta, dict) | ||||
| assert len(orig_meta["results"]) == 1 | assert len(orig_meta["results"]) == 1 | ||||
| assert orig_meta["next_page_token"] is None | assert orig_meta["next_page_token"] is None | ||||
| orig_meta0 = orig_meta["results"][0] | orig_meta0 = orig_meta["results"][0] | ||||
| assert orig_meta0["authority"] == authority | assert orig_meta0.authority == authority | ||||
| assert orig_meta0["fetcher"] == fetcher | assert orig_meta0.fetcher == fetcher | ||||
| # Retrieve the information for deposit status update query to the deposit | # Retrieve the information for deposit status update query to the deposit | ||||
| urls = [ | urls = [ | ||||
| m | m | ||||
| for m in requests_mock_datadir.request_history | for m in requests_mock_datadir.request_history | ||||
| if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | ||||
| ] | ] | ||||
| ▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | assert revision["metadata"] == { | ||||
| "sha256": "474bf646aeeff6d945eb752b1a9f8a40f3d81a88909ee7bd2d08cc822aa361e6", # noqa | "sha256": "474bf646aeeff6d945eb752b1a9f8a40f3d81a88909ee7bd2d08cc822aa361e6", # noqa | ||||
| }, | }, | ||||
| "filename": "archive.zip", | "filename": "archive.zip", | ||||
| "length": 956830, | "length": 956830, | ||||
| } | } | ||||
| ], | ], | ||||
| } | } | ||||
| fetcher = { | fetcher = MetadataFetcher(name="swh-deposit", version="0.0.1",) | ||||
| "name": "swh-deposit", | |||||
| "version": "0.0.1", | |||||
| } | |||||
| authority = { | authority = MetadataAuthority( | ||||
| "type": "deposit_client", | type=MetadataAuthorityType.DEPOSIT, | ||||
| "url": "https://hal-test.archives-ouvertes.fr/", | url="https://hal-test.archives-ouvertes.fr/", | ||||
| } | ) | ||||
| # Check the metadata swh side | # Check the metadata swh side | ||||
| orig_meta = loader.storage.origin_metadata_get(url, authority) | orig_meta = loader.storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, url, authority | |||||
| ) | |||||
| assert orig_meta is not None | assert orig_meta is not None | ||||
| assert isinstance(orig_meta, dict) | assert isinstance(orig_meta, dict) | ||||
| assert len(orig_meta["results"]) == 1 | assert len(orig_meta["results"]) == 1 | ||||
| assert orig_meta["next_page_token"] is None | assert orig_meta["next_page_token"] is None | ||||
| assert len(orig_meta["results"]) == 1 | assert len(orig_meta["results"]) == 1 | ||||
| orig_meta0 = orig_meta["results"][0] | orig_meta0 = orig_meta["results"][0] | ||||
| # dynamic, a pain to display and not that interesting | |||||
| orig_meta0.pop("discovery_date") | |||||
| assert orig_meta0 == { | assert orig_meta0 == RawExtrinsicMetadata( | ||||
| "origin_url": "https://hal-test.archives-ouvertes.fr/some-external-id", | type=MetadataTargetType.ORIGIN, | ||||
| "metadata": json.dumps( | id="https://hal-test.archives-ouvertes.fr/some-external-id", | ||||
| discovery_date=orig_meta0.discovery_date, | |||||
| metadata=json.dumps( | |||||
| { | { | ||||
| "@xmlns": ["http://www.w3.org/2005/Atom"], | "@xmlns": ["http://www.w3.org/2005/Atom"], | ||||
| "author": ["some awesome author", "another one", "no one"], | "author": ["some awesome author", "another one", "no one"], | ||||
| "codemeta:dateCreated": "2017-10-07T15:17:08Z", | "codemeta:dateCreated": "2017-10-07T15:17:08Z", | ||||
| "codemeta:datePublished": "2017-10-08T15:00:00Z", | "codemeta:datePublished": "2017-10-08T15:00:00Z", | ||||
| "external_identifier": "some-external-id", | "external_identifier": "some-external-id", | ||||
| "url": "https://hal-test.archives-ouvertes.fr/some-external-id", | "url": "https://hal-test.archives-ouvertes.fr/some-external-id", | ||||
| } | } | ||||
| ).encode(), | ).encode(), | ||||
| "format": "sword-v2-atom-codemeta-v2-in-json", | format="sword-v2-atom-codemeta-v2-in-json", | ||||
| "authority": authority, | authority=authority, | ||||
| "fetcher": fetcher, | fetcher=fetcher, | ||||
| } | ) | ||||
| # Retrieve the information for deposit status update query to the deposit | # Retrieve the information for deposit status update query to the deposit | ||||
| urls = [ | urls = [ | ||||
| m | m | ||||
| for m in requests_mock_datadir.request_history | for m in requests_mock_datadir.request_history | ||||
| if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | if m.url == f"{DEPOSIT_URL}/{deposit_id}/update/" | ||||
| ] | ] | ||||
| Show All 13 Lines | |||||