diff --git a/swh/loader/package/deposit/loader.py b/swh/loader/package/deposit/loader.py --- a/swh/loader/package/deposit/loader.py +++ b/swh/loader/package/deposit/loader.py @@ -67,7 +67,7 @@ # which computes itself the values. The loader needs to use those to create the # release. - all_metadata_raw: List[str] = metadata["metadata_raw"] + metadata_raw: str = metadata["metadata_raw"] depo = metadata["deposit"] return cls( url=url, @@ -84,10 +84,9 @@ directory_extrinsic_metadata=[ RawExtrinsicMetadataCore( discovery_date=now(), - metadata=raw_metadata.encode(), + metadata=metadata_raw.encode(), format="sword-v2-atom-codemeta-v2", ) - for raw_metadata in all_metadata_raw ], ) @@ -209,10 +208,10 @@ def get_extrinsic_origin_metadata(self) -> List[RawExtrinsicMetadataCore]: metadata = self.metadata() - all_metadata_raw: List[str] = metadata["metadata_raw"] + metadata_raw: str = metadata["metadata_raw"] origin_metadata = json.dumps( { - "metadata": all_metadata_raw, + "metadata": [metadata_raw], "provider": metadata["provider"], "tool": metadata["tool"], } @@ -220,16 +219,14 @@ return [ RawExtrinsicMetadataCore( discovery_date=now(), - metadata=raw_meta.encode(), + metadata=metadata_raw.encode(), format="sword-v2-atom-codemeta-v2", - ) - for raw_meta in all_metadata_raw - ] + [ + ), RawExtrinsicMetadataCore( discovery_date=now(), metadata=origin_metadata, format="original-artifacts-json", - ) + ), ] @cached_method diff --git a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.10.json b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.10.json --- a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.10.json +++ b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.10.json @@ -3,17 +3,7 @@ "url": "https://hal-test.archives-ouvertes.fr/some-external-id", "type": "deposit" }, - "metadata_raw" : ["some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother one"], - "metadata_dict": { - "author": [ - "some awesome author", - "another one", - "no one" - ], - "codemeta:dateCreated": "2017-10-07T15:17:08Z", - "external_identifier": "some-external-id", - "url": "https://hal-test.archives-ouvertes.fr/some-external-id" - }, + "metadata_raw" : "some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother one", "provider": { "provider_name": "hal", "provider_type": "deposit_client", diff --git a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.11.json b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.11.json --- a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.11.json +++ b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.11.json @@ -3,20 +3,7 @@ "url": "https://hal-test.archives-ouvertes.fr/some-external-id", "type": "deposit" }, - "metadata_raw": ["some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", -"someone" - ], - "metadata_dict": { - "author": [ - "some awesome author", - "another one", - "no one" - ], - "codemeta:dateCreated": "2017-10-07T15:17:08Z", - "codemeta:datePublished": "2017-10-08T15:00:00Z", - "external_identifier": "some-external-id", - "url": "https://hal-test.archives-ouvertes.fr/some-external-id" - }, + "metadata_raw": "some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", "provider": { "provider_name": "hal", "provider_type": "deposit_client", diff --git a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.12.json b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.12.json --- a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.12.json +++ b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.12.json @@ -3,20 +3,7 @@ "url": "https://hal-test.archives-ouvertes.fr/hal-123456", "type": "deposit" }, - "metadata_raw": ["some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", -"someone" - ], - "metadata_dict": { - "author": [ - "some awesome author", - "another one", - "no one" - ], - "codemeta:dateCreated": "2017-10-07T15:17:08Z", - "codemeta:datePublished": "2017-10-08T15:00:00Z", - "external_identifier": "some-external-id", - "url": "https://hal-test.archives-ouvertes.fr/some-external-id" - }, + "metadata_raw": "some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", "provider": { "provider_name": "hal", "provider_type": "deposit_client", diff --git a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.13.json b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.13.json --- a/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.13.json +++ b/swh/loader/package/deposit/tests/data/https_deposit.softwareheritage.org/hello_2.13.json @@ -3,20 +3,7 @@ "url": "https://hal-test.archives-ouvertes.fr/hal-123456", "type": "deposit" }, - "metadata_raw": ["some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", -"someone\nThis release adds this and that." - ], - "metadata_dict": { - "author": [ - "some awesome author", - "another one", - "no one" - ], - "codemeta:dateCreated": "2017-10-07T15:17:08Z", - "codemeta:datePublished": "2017-10-08T15:00:00Z", - "external_identifier": "some-external-id", - "url": "https://hal-test.archives-ouvertes.fr/some-external-id" - }, + "metadata_raw": "some-external-idhttps://hal-test.archives-ouvertes.fr/some-external-id2017-10-07T15:17:08Zsome awesome authoranother oneno one", "provider": { "provider_name": "hal", "provider_type": "deposit_client", diff --git a/swh/loader/package/deposit/tests/test_deposit.py b/swh/loader/package/deposit/tests/test_deposit.py --- a/swh/loader/package/deposit/tests/test_deposit.py +++ b/swh/loader/package/deposit/tests/test_deposit.py @@ -6,7 +6,6 @@ import datetime import json import re -from typing import List import pytest @@ -230,9 +229,9 @@ ) assert orig_meta.next_page_token is None raw_meta = loader.client.metadata_get(deposit_id) - all_metadata_raw: List[str] = raw_meta["metadata_raw"] + metadata_raw: str = raw_meta["metadata_raw"] # 2 raw metadata xml + 1 json dict - assert len(orig_meta.results) == len(all_metadata_raw) + 1 + assert len(orig_meta.results) == 2 orig_meta0 = orig_meta.results[0] assert orig_meta0.authority == authority assert orig_meta0.fetcher == fetcher @@ -246,11 +245,11 @@ directory_swhid, authority ) assert actual_dir_meta.next_page_token is None - assert len(actual_dir_meta.results) == len(all_metadata_raw) - for dir_meta in actual_dir_meta.results: - assert dir_meta.authority == authority - assert dir_meta.fetcher == fetcher - assert dir_meta.metadata.decode() in all_metadata_raw + assert len(actual_dir_meta.results) == 1 + dir_meta = actual_dir_meta.results[0] + assert dir_meta.authority == authority + assert dir_meta.fetcher == fetcher + assert dir_meta.metadata.decode() == metadata_raw # Retrieve the information for deposit status update query to the deposit urls = [ @@ -366,28 +365,27 @@ Origin(url).swhid(), authority ) assert origin_extrinsic_metadata.next_page_token is None - all_metadata_raw: List[str] = raw_meta["metadata_raw"] + metadata_raw: str = raw_meta["metadata_raw"] # 1 raw metadata xml + 1 json dict - assert len(origin_extrinsic_metadata.results) == len(all_metadata_raw) + 1 + assert len(origin_extrinsic_metadata.results) == 2 origin_swhid = Origin(url).swhid() expected_metadata = [] - for idx, raw_meta in enumerate(all_metadata_raw): - origin_meta = origin_extrinsic_metadata.results[idx] - expected_metadata.append( - RawExtrinsicMetadata( - target=origin_swhid, - discovery_date=origin_meta.discovery_date, - metadata=raw_meta.encode(), - format="sword-v2-atom-codemeta-v2", - authority=authority, - fetcher=fetcher, - ) + origin_meta = origin_extrinsic_metadata.results[0] + expected_metadata.append( + RawExtrinsicMetadata( + target=origin_swhid, + discovery_date=origin_meta.discovery_date, + metadata=metadata_raw.encode(), + format="sword-v2-atom-codemeta-v2", + authority=authority, + fetcher=fetcher, ) + ) origin_metadata = { - "metadata": all_metadata_raw, + "metadata": [metadata_raw], "provider": provider, "tool": tool, } @@ -414,7 +412,7 @@ ) assert actual_directory_metadata.next_page_token is None - assert len(actual_directory_metadata.results) == len(all_metadata_raw) + assert len(actual_directory_metadata.results) == 1 release_swhid = CoreSWHID( object_type=ObjectType.RELEASE, object_id=hash_to_bytes(release_id) @@ -432,21 +430,20 @@ ) expected_directory_metadata = [] - for idx, raw_meta in enumerate(all_metadata_raw): - dir_metadata = actual_directory_metadata.results[idx] - expected_directory_metadata.append( - RawExtrinsicMetadata.from_dict( - { - **{ - k: v - for (k, v) in dir_metadata_template.to_dict().items() - if k != "id" - }, - "discovery_date": dir_metadata.discovery_date, - "metadata": raw_meta.encode(), - } - ) + dir_metadata = actual_directory_metadata.results[0] + expected_directory_metadata.append( + RawExtrinsicMetadata.from_dict( + { + **{ + k: v + for (k, v) in dir_metadata_template.to_dict().items() + if k != "id" + }, + "discovery_date": dir_metadata.discovery_date, + "metadata": metadata_raw.encode(), + } ) + ) assert sorted(actual_directory_metadata.results) == sorted( expected_directory_metadata