Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/migrate_extrinsic_metadata/test_deposit.py
- This file was added.
# Copyright (C) 2020 The Software Heritage developers | |||||
# See the AUTHORS file at the top-level directory of this distribution | |||||
# License: GNU General Public License version 3, or any later version | |||||
# See top-level LICENSE file for more information | |||||
# flake8: noqa | |||||
# because of long lines | |||||
import copy | |||||
import datetime | |||||
import json | |||||
from unittest.mock import call, Mock, MagicMock | |||||
from swh.model.identifiers import parse_swhid | |||||
from swh.model.model import ( | |||||
MetadataAuthority, | |||||
MetadataAuthorityType, | |||||
MetadataFetcher, | |||||
MetadataTargetType, | |||||
Origin, | |||||
RawExtrinsicMetadata, | |||||
) | |||||
from swh.storage.migrate_extrinsic_metadata import ( | |||||
DEPOSIT_COLS, | |||||
handle_row, | |||||
cran_package_from_url, | |||||
) | |||||
FETCHER = MetadataFetcher( | |||||
name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", | |||||
) | |||||
SWH_AUTHORITY = MetadataAuthority( | |||||
type=MetadataAuthorityType.REGISTRY, | |||||
url="https://softwareheritage.org/", | |||||
metadata={}, | |||||
) | |||||
SWH_DEPOSIT_AUTHORITY = MetadataAuthority( | |||||
type=MetadataAuthorityType.DEPOSIT_CLIENT, | |||||
url="https://www.softwareheritage.org", | |||||
metadata={}, | |||||
) | |||||
HAL_AUTHORITY = MetadataAuthority( | |||||
type=MetadataAuthorityType.DEPOSIT_CLIENT, | |||||
url="https://hal.archives-ouvertes.fr/", | |||||
metadata={}, | |||||
) | |||||
INTEL_AUTHORITY = MetadataAuthority( | |||||
type=MetadataAuthorityType.DEPOSIT_CLIENT, | |||||
url="https://software.intel.com", | |||||
metadata={}, | |||||
) | |||||
def get_mock_deposit_cur(row_dicts): | |||||
rows = [tuple(d[key] for key in DEPOSIT_COLS) for d in row_dicts] | |||||
deposit_cur = MagicMock() | |||||
deposit_cur.__iter__.side_effect = [iter(rows)] | |||||
return deposit_cur | |||||
def test_deposit_1(): | |||||
"""Has a provider and xmlns, and the metadata is in the revision twice | |||||
(at the root of the metadata dict, and in | |||||
metadata->extrinsic->raw->origin_metadata)""" | |||||
extrinsic_metadata = { | |||||
"title": "Je suis GPL", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"client": "swh", | |||||
"codemeta:url": "https://forge.softwareheritage.org/source/jesuisgpl/", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": { | |||||
"codemeta:name": "Stefano Zacchiroli", | |||||
"codemeta:jobTitle": "Maintainer", | |||||
}, | |||||
"codemeta:license": { | |||||
"codemeta:url": "https://spdx.org/licenses/GPL-3.0-or-later.html", | |||||
"codemeta:name": "GNU General Public License v3.0 or later", | |||||
}, | |||||
# ... | |||||
} | |||||
original_artifacts = [ | |||||
{ | |||||
"length": 80880, | |||||
"filename": "archive.zip", | |||||
"checksums": { | |||||
"sha1": "bad32a47a359e0e16ebdca2ad2dc6a771dac8f71", | |||||
"sha256": "182b7ee3b7b5b550e83d3bcfed029bb2f625ee760ebfe9557d5fd072bd4e22e4", | |||||
}, | |||||
} | |||||
] | |||||
row = { | |||||
"id": b"\x02#\x10\xdf\x16\xfd\x9eMO\x81\xfe6\xa1B\xe8-\xb9w\xc0\x1d", | |||||
"date": datetime.datetime(2018, 1, 5, 0, 0, tzinfo=datetime.timezone.utc), | |||||
"committer_date": datetime.datetime( | |||||
2018, 1, 5, 0, 0, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"swh: Deposit 467 in collection swh", | |||||
"metadata": { | |||||
"client": "swh", | |||||
"extrinsic": { | |||||
"raw": { | |||||
"origin": { | |||||
"url": "https://www.softwareheritage.org/check-deposit-2020-03-11T11:07:18.424476", | |||||
"type": "deposit", | |||||
}, | |||||
"branch_name": "master", | |||||
"origin_metadata": { | |||||
"tool": { | |||||
"name": "swh-deposit", | |||||
"version": "0.0.1", | |||||
"configuration": {"sword_version": 2}, | |||||
}, | |||||
"metadata": extrinsic_metadata, | |||||
}, | |||||
}, | |||||
"when": "2020-03-11T11:11:36.336283+00:00", | |||||
"provider": "https://deposit.softwareheritage.org/1/private/467/meta/", | |||||
}, | |||||
"original_artifact": original_artifacts, | |||||
**extrinsic_metadata, | |||||
}, | |||||
} | |||||
origin_url = ( | |||||
"https://www.softwareheritage.org/check-deposit-2020-03-11T11:07:18.424476" | |||||
) | |||||
swhid = ( | |||||
f"swh:1:dir:ef04a768181417fbc5eef4243e2507915f24deea" | |||||
f";origin={origin_url}" | |||||
f";visit=swh:1:snp:14433c19dbb03ad57c86b58b53a800d6a0e32dd3" | |||||
f";anchor=swh:1:rev:022310df16fd9e4d4f81fe36a142e82db977c01d" | |||||
f";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 467, | |||||
"deposit.external_id": "check-deposit-2020-03-11T11:07:18.424476", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2020, 3, 11, 11, 7, 18, 688410, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://www.softwareheritage.org", | |||||
"deposit_collection.name": "swh", | |||||
"auth_user.username": "swh", | |||||
}, | |||||
{ | |||||
"deposit.id": 467, | |||||
"deposit.external_id": "check-deposit-2020-03-11T11:07:18.424476", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2020, 3, 11, 11, 7, 18, 669428, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://www.softwareheritage.org", | |||||
"deposit_collection.name": "swh", | |||||
"auth_user.username": "swh", | |||||
}, | |||||
] | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:022310df16fd9e4d4f81fe36a142e82db977c01d" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2020, 3, 11, 11, 7, 18, 688410, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_DEPOSIT_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:022310df16fd9e4d4f81fe36a142e82db977c01d" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2020, 3, 11, 11, 11, 36, 336283, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="original-artifacts-json", | |||||
metadata=json.dumps(original_artifacts).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
] | |||||
def test_deposit_2_without_xmlns(): | |||||
"""Has a provider, no xmlns, and the metadata is only in | |||||
metadata->extrinsic->raw->origin_metadata)""" | |||||
extrinsic_metadata = { | |||||
"{http://www.w3.org/2005/Atom}id": "hal-01243573", | |||||
"{http://www.w3.org/2005/Atom}author": { | |||||
"{http://www.w3.org/2005/Atom}name": "HAL", | |||||
"{http://www.w3.org/2005/Atom}email": "hal@ccsd.cnrs.fr", | |||||
}, | |||||
"{http://www.w3.org/2005/Atom}client": "hal", | |||||
"{http://www.w3.org/2005/Atom}external_identifier": "hal-01243573", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}url": "https://hal-test.archives-ouvertes.fr/hal-01243573", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "The assignment problem", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}author": { | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "Morane Gruenpeter" | |||||
}, | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}version": 1, | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}identifier": "10.5281/zenodo.438684", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}dateCreated": "2017-11-16T14:54:23+01:00", | |||||
} | |||||
original_artifacts = [ | |||||
{ | |||||
"length": 208357, | |||||
"filename": "archive.zip", | |||||
"checksums": { | |||||
"sha1": "fa0aec08e8a44ea144dba7ce366c8b5d66c14453", | |||||
"sha256": "f53c05fe947e88ce83751a93bd522b1f88478ea2e7b984c07fc7a7c68128bf87", | |||||
}, | |||||
} | |||||
] | |||||
row = { | |||||
"id": b"\x01\x16\xca\xb7\x19d\xd5\x9c\x85p\xb4\xc5r\x9b(\xbd\xd6<\x9bF", | |||||
"date": datetime.datetime( | |||||
2018, 1, 17, 12, 54, 0, 723882, tzinfo=datetime.timezone.utc | |||||
), | |||||
"committer_date": datetime.datetime( | |||||
2018, 1, 17, 12, 54, 0, 723882, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"hal: Deposit 82 in collection hal", | |||||
"metadata": { | |||||
"extrinsic": { | |||||
"raw": { | |||||
"origin": { | |||||
"url": "https://hal.archives-ouvertes.fr/hal-01243573", | |||||
"type": "deposit", | |||||
}, | |||||
"origin_metadata": { | |||||
"tool": { | |||||
"name": "swh-deposit", | |||||
"version": "0.0.1", | |||||
"configuration": {"sword_version": 2}, | |||||
}, | |||||
"metadata": extrinsic_metadata, | |||||
"provider": { | |||||
"metadata": {}, | |||||
"provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"provider_name": "hal", | |||||
"provider_type": "deposit_client", | |||||
}, | |||||
}, | |||||
}, | |||||
"when": "2020-05-15T14:27:21.462270+00:00", | |||||
"provider": "https://deposit.softwareheritage.org/1/private/82/meta/", | |||||
}, | |||||
"original_artifact": original_artifacts, | |||||
}, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:e04b2a7b8a8838da0693e9fd992a10d6fd211b50" | |||||
";origin=https://hal.archives-ouvertes.fr/hal-01243573" | |||||
";visit=swh:1:snp:abc9ae594245a740235b6c039f044352a5f723ec" | |||||
";anchor=swh:1:rev:0116cab71964d59c8570b4c5729b28bdd63c9b46" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 82, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 17, 12, 54, 1, 533972, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
{ | |||||
"deposit.id": 82, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 17, 12, 54, 0, 413748, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
] | |||||
origin_url = "https://hal.archives-ouvertes.fr/hal-01243573" | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:0116cab71964d59c8570b4c5729b28bdd63c9b46" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2018, 1, 17, 12, 54, 0, 413748, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=HAL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json-with-expanded-namespaces", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:0116cab71964d59c8570b4c5729b28bdd63c9b46" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2020, 5, 15, 14, 27, 21, 462270, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="original-artifacts-json", | |||||
metadata=json.dumps(original_artifacts).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
] | |||||
def test_deposit_2_with_xmlns(): | |||||
"""Has a provider, xmlns, and the metadata is only in | |||||
metadata->extrinsic->raw->origin_metadata)""" | |||||
extrinsic_metadata = { | |||||
"title": "Je suis GPL", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"client": "swh", | |||||
"codemeta:url": "https://forge.softwareheritage.org/source/jesuisgpl/", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": { | |||||
"codemeta:name": "Stefano Zacchiroli", | |||||
"codemeta:jobTitle": "Maintainer", | |||||
}, | |||||
"codemeta:license": { | |||||
"codemeta:url": "https://spdx.org/licenses/GPL-3.0-or-later.html", | |||||
"codemeta:name": "GNU General Public License v3.0 or later", | |||||
}, | |||||
"external_identifier": "je-suis-gpl", | |||||
"codemeta:dateCreated": "2018-01-05", | |||||
} | |||||
original_artifacts = [ | |||||
{ | |||||
"length": 80880, | |||||
"filename": "archive.zip", | |||||
"checksums": { | |||||
"sha1": "bad32a47a359e0e16ebdca2ad2dc6a771dac8f71", | |||||
"sha256": "182b7ee3b7b5b550e83d3bcfed029bb2f625ee760ebfe9557d5fd072bd4e22e4", | |||||
}, | |||||
} | |||||
] | |||||
row = { | |||||
"id": b'\x01"\x96nP\x93\x17\xae\xcejA\xd0\xf0\x88\xdas<\xc0\x9d\x0f', | |||||
"date": datetime.datetime(2018, 1, 5, 0, 0, tzinfo=datetime.timezone.utc), | |||||
"committer_date": datetime.datetime( | |||||
2018, 1, 5, 0, 0, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"swh: Deposit 687 in collection swh", | |||||
"metadata": { | |||||
"extrinsic": { | |||||
"raw": { | |||||
"origin": { | |||||
"url": "https://www.softwareheritage.org/check-deposit-2020-06-26T13:50:07.564420", | |||||
"type": "deposit", | |||||
}, | |||||
"origin_metadata": { | |||||
"tool": { | |||||
"name": "swh-deposit", | |||||
"version": "0.0.1", | |||||
"configuration": {"sword_version": 2}, | |||||
}, | |||||
"metadata": extrinsic_metadata, | |||||
"provider": { | |||||
"metadata": {}, | |||||
"provider_url": "https://www.softwareheritage.org", | |||||
"provider_name": "swh", | |||||
"provider_type": "deposit_client", | |||||
}, | |||||
}, | |||||
}, | |||||
"when": "2020-06-26T13:50:22.640625+00:00", | |||||
"provider": "https://deposit.softwareheritage.org/1/private/687/meta/", | |||||
}, | |||||
"original_artifact": original_artifacts, | |||||
}, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:ef04a768181417fbc5eef4243e2507915f24deea" | |||||
";origin=https://www.softwareheritage.org/check-deposit-2020-06-26T13:50:07.564420" | |||||
";visit=swh:1:snp:8fd469e280fb0724175c64906627f619143d5bdb" | |||||
";anchor=swh:1:rev:0122966e509317aece6a41d0f088da733cc09d0f" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 687, | |||||
"deposit.external_id": "check-deposit-2020-06-26T13:50:07.564420", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2020, 6, 26, 13, 50, 8, 216113, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://www.softwareheritage.org", | |||||
"deposit_collection.name": "swh", | |||||
"auth_user.username": "swh", | |||||
}, | |||||
{ | |||||
"deposit.id": 687, | |||||
"deposit.external_id": "check-deposit-2020-06-26T13:50:07.564420", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2020, 6, 26, 13, 50, 8, 150498, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://www.softwareheritage.org", | |||||
"deposit_collection.name": "swh", | |||||
"auth_user.username": "swh", | |||||
}, | |||||
] | |||||
origin_url = ( | |||||
"https://www.softwareheritage.org/check-deposit-2020-06-26T13:50:07.564420" | |||||
) | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:0122966e509317aece6a41d0f088da733cc09d0f" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2020, 6, 26, 13, 50, 8, 216113, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_DEPOSIT_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:0122966e509317aece6a41d0f088da733cc09d0f" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2020, 6, 26, 13, 50, 22, 640625, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="original-artifacts-json", | |||||
metadata=json.dumps(original_artifacts).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
] | |||||
def test_deposit_3_and_wrong_external_id_in_metadata(): | |||||
extrinsic_metadata = { | |||||
"title": "VTune Perf tool", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"client": "swh", | |||||
"codemeta:url": "https://software.intel.com/en-us/vtune", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": { | |||||
"codemeta:name": "VTune developer", | |||||
"codemeta:jobTitle": "Software Engineer", | |||||
}, | |||||
"external_identifier": "vtune-perf-tool", | |||||
"codemeta:dateCreated": "2019-05-14", | |||||
"codemeta:description": "Modified version of Linux Perf tool which is used by Intel VTune Amplifier", | |||||
} | |||||
source_original_artifacts = [ | |||||
{ | |||||
"name": "archive.zip", | |||||
"sha1": "07251dbb1d904d143fd7da9935701f17670d4d9b", | |||||
"length": 4350528, | |||||
"sha256": "1f7d111ac79e468002f3edf4b7b2487538d41f6bea362d49b2eb08a537efafb6", | |||||
"sha1_git": "e2d894efcaad4ff36f09eda3b3c0096416b03429", | |||||
"blake2s256": "e2c08b82efbc361fbb2d28aa8352668cd71217f165f63de16b61ed61ace7509d", | |||||
"archive_type": "zip", | |||||
} | |||||
] | |||||
dest_original_artifacts = [ | |||||
{ | |||||
"length": 4350528, | |||||
"archive_type": "zip", | |||||
"filename": "archive.zip", | |||||
"checksums": { | |||||
"sha1": "07251dbb1d904d143fd7da9935701f17670d4d9b", | |||||
"sha256": "1f7d111ac79e468002f3edf4b7b2487538d41f6bea362d49b2eb08a537efafb6", | |||||
"sha1_git": "e2d894efcaad4ff36f09eda3b3c0096416b03429", | |||||
"blake2s256": "e2c08b82efbc361fbb2d28aa8352668cd71217f165f63de16b61ed61ace7509d", | |||||
}, | |||||
} | |||||
] | |||||
row = { | |||||
"id": b"\t5`S\xc4\x9a\xd0\xf9\xe6.Q\xc2\x9d>a|y\x11@\xdf", | |||||
"date": datetime.datetime(2019, 5, 14, 0, 0, tzinfo=datetime.timezone.utc), | |||||
"committer_date": datetime.datetime( | |||||
2019, 5, 14, 0, 0, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"intel: Deposit 268 in collection intel", | |||||
"metadata": { | |||||
**extrinsic_metadata, | |||||
"original_artifact": source_original_artifacts, | |||||
}, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:527c8e4a67d391f2bf1bbc86dd94af5d5cfc8ef7" | |||||
";origin=https://software.intel.com/f80482de-90a8-4c32-bce4-6f6918d492ff" | |||||
";visit=swh:1:snp:49d60943d9c061da1aba6266a811412f9db8de2e" | |||||
";anchor=swh:1:rev:09356053c49ad0f9e62e51c29d3e617c791140df" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 268, | |||||
"deposit.external_id": "f80482de-90a8-4c32-bce4-6f6918d492ff", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 5, 14, 7, 49, 36, 775072, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://software.intel.com", | |||||
"deposit_collection.name": "intel", | |||||
"auth_user.username": "intel", | |||||
}, | |||||
{ | |||||
"deposit.id": 268, | |||||
"deposit.external_id": "f80482de-90a8-4c32-bce4-6f6918d492ff", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 5, 14, 7, 49, 36, 477061, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://software.intel.com", | |||||
"deposit_collection.name": "intel", | |||||
"auth_user.username": "intel", | |||||
}, | |||||
{ | |||||
"deposit.id": 268, | |||||
"deposit.external_id": "f80482de-90a8-4c32-bce4-6f6918d492ff", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 5, 14, 7, 28, 33, 210100, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://software.intel.com", | |||||
"deposit_collection.name": "intel", | |||||
"auth_user.username": "intel", | |||||
}, | |||||
{ | |||||
"deposit.id": 268, | |||||
"deposit.external_id": "f80482de-90a8-4c32-bce4-6f6918d492ff", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 5, 14, 7, 28, 33, 41454, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://software.intel.com", | |||||
"deposit_collection.name": "intel", | |||||
"auth_user.username": "intel", | |||||
}, | |||||
] | |||||
origin_url = "https://software.intel.com/f80482de-90a8-4c32-bce4-6f6918d492ff" | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:09356053c49ad0f9e62e51c29d3e617c791140df" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2019, 5, 14, 7, 49, 36, 775072, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=INTEL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:09356053c49ad0f9e62e51c29d3e617c791140df" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2019, 5, 14, 7, 28, 33, 210100, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=INTEL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:09356053c49ad0f9e62e51c29d3e617c791140df" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2019, 5, 14, 7, 49, 36, 775072, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="original-artifacts-json", | |||||
metadata=json.dumps(dest_original_artifacts).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
] | |||||
def test_deposit_3_and_no_swhid(): | |||||
extrinsic_metadata = { | |||||
"id": "hal-02337300", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"author": {"name": "HAL", "email": "hal@ccsd.cnrs.fr"}, | |||||
"client": "hal", | |||||
"codemeta:url": "https://hal.archives-ouvertes.fr/hal-02337300", | |||||
"codemeta:name": "R package SMM, Simulation and Estimation of Multi-State Discrete-Time Semi-Markov and Markov Models", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": [ | |||||
# ... | |||||
], | |||||
# ... | |||||
} | |||||
original_artifacts = [ | |||||
# ... | |||||
] | |||||
row = { | |||||
"id": b"\x91\xe5\xca\x8b'K\xf1\xa8cFd2\xd7Q\xf7A\xbc\x94\xba&", | |||||
"date": datetime.datetime(2017, 1, 1, 0, 0, tzinfo=datetime.timezone.utc), | |||||
"committer_date": datetime.datetime( | |||||
2019, 11, 6, 14, 47, 30, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"hal: Deposit 342 in collection hal", | |||||
"metadata": {**extrinsic_metadata, "original_artifact": original_artifacts,}, | |||||
} | |||||
storage = Mock() | |||||
deposit_cur = None | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
assert storage.method_calls == [] | |||||
def test_deposit_3_and_unknown_deposit(): | |||||
extrinsic_metadata = { | |||||
"title": "Je suis GPL", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"client": "swh", | |||||
"codemeta:url": "https://forge.softwareheritage.org/source/jesuisgpl/", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": { | |||||
"codemeta:name": "Stefano Zacchiroli", | |||||
"codemeta:jobTitle": "Maintainer", | |||||
}, | |||||
# ... | |||||
} | |||||
row = { | |||||
"id": b"\x8e\x9c\xee\x14\xa6\xad9\xbc\xa44pw\xb8\x7f\xb5\xbb\xd8\x95;\xb1", | |||||
"date": datetime.datetime( | |||||
2018, 7, 23, 12, 25, 45, 907132, tzinfo=datetime.timezone.utc | |||||
), | |||||
"committer_date": datetime.datetime( | |||||
2018, 7, 23, 12, 25, 45, 907132, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"swh: Deposit 159 in collection swh", | |||||
"metadata": extrinsic_metadata, | |||||
} | |||||
origin_url = "https://software.intel.com/f80482de-90a8-4c32-bce4-6f6918d492ff" | |||||
storage = Mock() | |||||
deposit_cur = None | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
assert storage.method_calls == [] | |||||
def test_deposit_4_without_xmlns(): | |||||
extrinsic_metadata = { | |||||
"{http://www.w3.org/2005/Atom}id": "hal-01243573", | |||||
"{http://www.w3.org/2005/Atom}author": { | |||||
"{http://www.w3.org/2005/Atom}name": "HAL", | |||||
"{http://www.w3.org/2005/Atom}email": "hal@ccsd.cnrs.fr", | |||||
}, | |||||
"{http://www.w3.org/2005/Atom}client": "hal", | |||||
"{http://www.w3.org/2005/Atom}external_identifier": "hal-01243573", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}url": "https://hal-test.archives-ouvertes.fr/hal-01243573", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "The assignment problem", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}author": { | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "Morane Gruenpeter" | |||||
}, | |||||
# ... | |||||
} | |||||
row = { | |||||
"id": b"\x03\x98\x7f\x05n\xafE\x96\xcd \xd7\xb2\xee\x01\xc9\xb8L\xed\xdf\xa8", | |||||
"date": datetime.datetime( | |||||
2018, 1, 17, 12, 49, 30, 902891, tzinfo=datetime.timezone.utc | |||||
), | |||||
"committer_date": datetime.datetime( | |||||
2018, 1, 17, 12, 49, 30, 902891, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b": Deposit 79 in collection hal", | |||||
"metadata": extrinsic_metadata, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:e04b2a7b8a8838da0693e9fd992a10d6fd211b50" | |||||
";origin=https://hal.archives-ouvertes.fr/hal-01243573" | |||||
";visit=swh:1:snp:c31851534c86676a040fb10f438728c90f1c9d55" | |||||
";anchor=swh:1:rev:43549ebbe70c9cdf0be1647e6319392eaa06f3a3" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 79, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 17, 12, 49, 31, 208347, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
{ | |||||
"deposit.id": 79, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 17, 12, 49, 30, 645576, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
] | |||||
origin_url = "https://hal.archives-ouvertes.fr/hal-01243573" | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:03987f056eaf4596cd20d7b2ee01c9b84ceddfa8" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2018, 1, 17, 12, 49, 30, 645576, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=HAL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json-with-expanded-namespaces", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
# note: no original artifacts | |||||
] | |||||
def test_deposit_4_wrong_origin(): | |||||
extrinsic_metadata = { | |||||
"{http://www.w3.org/2005/Atom}id": "hal-01588781", | |||||
"{http://www.w3.org/2005/Atom}author": { | |||||
"{http://www.w3.org/2005/Atom}name": "HAL", | |||||
"{http://www.w3.org/2005/Atom}email": "hal@ccsd.cnrs.fr", | |||||
}, | |||||
"{http://www.w3.org/2005/Atom}client": "hal", | |||||
"{http://www.w3.org/2005/Atom}external_identifier": "hal-01588781", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}url": "https://inria.halpreprod.archives-ouvertes.fr/hal-01588781", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "The assignment problem ", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}author": { | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}name": "Morane Gruenpeter", | |||||
"{https://doi.org/10.5063/SCHEMA/CODEMETA-2.0}affiliation": "Initiative pour la Recherche et l'Innovation sur le Logiciel Libre", | |||||
}, | |||||
# ... | |||||
} | |||||
row = { | |||||
"id": b"-{\xcec\x1f\xc7\x91\x08\x03\x11\xeb\x83\\GB\x8eXjn\xa4", | |||||
"date": datetime.datetime( | |||||
2018, 1, 10, 13, 14, 51, 77033, tzinfo=datetime.timezone.utc | |||||
), | |||||
"committer_date": datetime.datetime( | |||||
2018, 1, 10, 13, 14, 51, 77033, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b": Deposit 75 in collection hal", | |||||
"metadata": extrinsic_metadata, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:d8971c651fe256942aa4499a3ccdbaa305d3bade" | |||||
";origin=https://inria.halpreprod.archives-ouvertes.fr/hal-01588781" | |||||
";visit=swh:1:snp:7c70cc8ea5b79e376605fd6e9b3b04d98861ffc0" | |||||
";anchor=swh:1:rev:2d7bce631fc791080311eb835c47428e586a6ea4" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 75, | |||||
"deposit.external_id": "hal-01588781", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 10, 13, 14, 51, 523963, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
{ | |||||
"deposit.id": 75, | |||||
"deposit.external_id": "hal-01588781", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2018, 1, 10, 13, 14, 50, 555143, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
] | |||||
origin_url = "https://inria.halpreprod.archives-ouvertes.fr/hal-01588781" | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:2d7bce631fc791080311eb835c47428e586a6ea4" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2018, 1, 10, 13, 14, 50, 555143, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=HAL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json-with-expanded-namespaces", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
# note: no original artifacts | |||||
] | |||||
def test_deposit_missing_metadata_in_revision(): | |||||
extrinsic_metadata = { | |||||
"id": "hal-01243573", | |||||
"@xmlns": "http://www.w3.org/2005/Atom", | |||||
"author": {"name": "HAL", "email": "hal@ccsd.cnrs.fr"}, | |||||
"client": "hal", | |||||
"committer": "Administrateur Du Ccsd", | |||||
"codemeta:url": "https://hal-test.archives-ouvertes.fr/hal-01243573", | |||||
"codemeta:name": "The assignment problem", | |||||
"@xmlns:codemeta": "https://doi.org/10.5063/SCHEMA/CODEMETA-2.0", | |||||
"codemeta:author": {"codemeta:name": "Morane Gruenpeter"}, | |||||
"codemeta:version": "1", | |||||
"codemeta:identifier": {"#text": "10.5281/zenodo.438684", "@name": "doi",}, | |||||
"external_identifier": "hal-01243573", | |||||
"codemeta:dateCreated": "2017-11-16T14:54:23+01:00", | |||||
} | |||||
source_original_artifacts = [ | |||||
{ | |||||
"name": "archive.zip", | |||||
"sha1": "e8e46324970cd5af7f98c5a86f33f47fa4a41b4a", | |||||
"length": 118650, | |||||
"sha256": "fec81b63d666c43524f966bbd3263da5bee55051d2b48c1659cca5f56fd953e5", | |||||
"sha1_git": "9da2bbd08bec590b36ede2ed43d74cd510b10a79", | |||||
"blake2s256": "5d0973ba3644cc2bcfdb41ff1891744337d6aa9547a7e59fe466f684b027f295", | |||||
"archive_type": "zip", | |||||
} | |||||
] | |||||
dest_original_artifacts = [ | |||||
{ | |||||
"length": 118650, | |||||
"archive_type": "zip", | |||||
"filename": "archive.zip", | |||||
"checksums": { | |||||
"sha1": "e8e46324970cd5af7f98c5a86f33f47fa4a41b4a", | |||||
"sha256": "fec81b63d666c43524f966bbd3263da5bee55051d2b48c1659cca5f56fd953e5", | |||||
"sha1_git": "9da2bbd08bec590b36ede2ed43d74cd510b10a79", | |||||
"blake2s256": "5d0973ba3644cc2bcfdb41ff1891744337d6aa9547a7e59fe466f684b027f295", | |||||
}, | |||||
} | |||||
] | |||||
row = { | |||||
"id": b"\x03@v\xf3\xf4\x1e\xe1 N\xb9\xf6@\x82\xcb\xe6\xe9P\xd7\xbb\x8a", | |||||
"date": datetime.datetime( | |||||
2019, 2, 25, 15, 49, 16, 594536, tzinfo=datetime.timezone.utc | |||||
), | |||||
"committer_date": datetime.datetime( | |||||
2019, 2, 25, 15, 49, 16, 594536, tzinfo=datetime.timezone.utc | |||||
), | |||||
"type": "tar", | |||||
"message": b"hal: Deposit 229 in collection hal", | |||||
"metadata": {"original_artifact": source_original_artifacts}, | |||||
} | |||||
swhid = ( | |||||
"swh:1:dir:3d65b6f065118cb856272829b459f0dfa55549aa" | |||||
";origin=https://hal-test.archives-ouvertes.fr/hal-01243573" | |||||
";visit=swh:1:snp:322c54ff4023d3216a994bc9ff9ee524ed80ee1f" | |||||
";anchor=swh:1:rev:034076f3f41ee1204eb9f64082cbe6e950d7bb8a" | |||||
";path=/" | |||||
) | |||||
deposit_rows = [ | |||||
{ | |||||
"deposit.id": 229, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": None, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 2, 25, 15, 54, 30, 102072, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
{ | |||||
"deposit.id": 229, | |||||
"deposit.external_id": "hal-01243573", | |||||
"deposit.swh_id_context": swhid, | |||||
"deposit.status": "success", | |||||
"deposit_request.metadata": extrinsic_metadata, | |||||
"deposit_request.date": datetime.datetime( | |||||
2019, 2, 25, 15, 49, 12, 302745, tzinfo=datetime.timezone.utc | |||||
), | |||||
"deposit_client.provider_url": "https://hal.archives-ouvertes.fr/", | |||||
"deposit_collection.name": "hal", | |||||
"auth_user.username": "hal", | |||||
}, | |||||
] | |||||
origin_url = "https://hal.archives-ouvertes.fr/hal-01243573" | |||||
# /!\ not https://hal-test.archives-ouvertes.fr/hal-01243573 | |||||
# do not trust the metadata! | |||||
storage = Mock() | |||||
def origin_get(urls): | |||||
assert urls == [origin_url] | |||||
return [Origin(url=origin_url)] | |||||
storage.origin_get.side_effect = origin_get | |||||
deposit_cur = get_mock_deposit_cur(deposit_rows) | |||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | |||||
deposit_cur.execute.assert_called_once() | |||||
deposit_cur.__iter__.assert_called_once() | |||||
assert storage.method_calls == [ | |||||
call.origin_get([origin_url]), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:034076f3f41ee1204eb9f64082cbe6e950d7bb8a" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2019, 2, 25, 15, 49, 12, 302745, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=HAL_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="sword-v2-atom-codemeta-v2-in-json", | |||||
metadata=json.dumps(extrinsic_metadata).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
call.raw_extrinsic_metadata_add( | |||||
[ | |||||
RawExtrinsicMetadata( | |||||
type=MetadataTargetType.REVISION, | |||||
id=parse_swhid( | |||||
"swh:1:rev:034076f3f41ee1204eb9f64082cbe6e950d7bb8a" | |||||
), | |||||
discovery_date=datetime.datetime( | |||||
2019, 2, 25, 15, 54, 30, 102072, tzinfo=datetime.timezone.utc | |||||
), | |||||
authority=SWH_AUTHORITY, | |||||
fetcher=FETCHER, | |||||
format="original-artifacts-json", | |||||
metadata=json.dumps(dest_original_artifacts).encode(), | |||||
origin=origin_url, | |||||
), | |||||
] | |||||
), | |||||
] |