diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py new file mode 100644 --- /dev/null +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py @@ -0,0 +1,150 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +# flake8: noqa +# because of long lines + +import datetime +import json +from unittest.mock import call, Mock + +from swh.model.identifiers import parse_swhid +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + MetadataFetcher, + MetadataTargetType, + Origin, + RawExtrinsicMetadata, +) + +from swh.storage.migrate_extrinsic_metadata import handle_row + + +FETCHER = MetadataFetcher( + name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", +) +PYPI_AUTHORITY = MetadataAuthority( + type=MetadataAuthorityType.FORGE, url="https://pypi.org/", metadata={}, +) +SWH_AUTHORITY = MetadataAuthority( + type=MetadataAuthorityType.REGISTRY, + url="https://softwareheritage.org/", + metadata={}, +) + + +def test_pypi_1(): + extrinsic_metadata = { + "url": "https://files.pythonhosted.org/packages/70/89/a498245baf1bf3dde73d3da00b4b067a8aa7c7378ad83472078803ea3e43/m3-ui-2.2.73.tar.gz", + "size": 3933168, + "digests": { + "md5": "a374ac3f655e97df5db5335e2142d344", + "sha256": "1bc2756f7d0d2e15cf5880ca697682ff35e8b58116bf73eb9c78b3db358c5b7d", + }, + "has_sig": False, + "filename": "m3-ui-2.2.73.tar.gz", + "downloads": -1, + "md5_digest": "a374ac3f655e97df5db5335e2142d344", + "packagetype": "sdist", + "upload_time": "2019-11-11T06:21:20", + "comment_text": "", + "python_version": "source", + "requires_python": None, + "upload_time_iso_8601": "2019-11-11T06:21:20.073082Z", + } + + original_artifact = [ + { + "length": 3933168, + "filename": "m3-ui-2.2.73.tar.gz", + "checksums": { + "sha1": "9f4ec7ce64b7fea4b122e85d47ea31146c367b03", + "sha256": "1bc2756f7d0d2e15cf5880ca697682ff35e8b58116bf73eb9c78b3db358c5b7d", + }, + } + ] + + row = { + "id": b"\x00\x00\x07a{S\xe7\xb1E\x8fi]\xd0}\xe4\xceU\xaf\x15\x17", + "date": datetime.datetime( + 2019, 11, 11, 6, 21, 20, tzinfo=datetime.timezone.utc, + ), + "committer_date": datetime.datetime( + 2019, 11, 11, 6, 21, 20, tzinfo=datetime.timezone.utc, + ), + "type": "tar", + "message": b"2.2.73", + "metadata": { + "extrinsic": { + "raw": extrinsic_metadata, + "when": "2020-01-23T18:43:09.109407+00:00", + "provider": "https://pypi.org/pypi/m3-ui/json", + }, + "intrinsic": { + "raw": { + "name": "m3-ui", + "summary": "======", + "version": "2.2.73", + # ... + "metadata_version": "1.1", + }, + "tool": "PKG-INFO", + }, + "original_artifact": original_artifact, + }, + } + + origin_url = "https://pypi.org/project/m3-ui/" + + storage = Mock() + + def origin_get(urls): + assert urls == [origin_url] + return [Origin(url=origin_url)] + + storage.origin_get.side_effect = origin_get + deposit_cur = None + handle_row(row, storage, deposit_cur, dry_run=False) + + assert storage.method_calls == [ + call.origin_get([origin_url]), + call.raw_extrinsic_metadata_add( + [ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=parse_swhid( + "swh:1:rev:000007617b53e7b1458f695dd07de4ce55af1517" + ), + discovery_date=datetime.datetime( + 2019, 11, 11, 6, 21, 20, tzinfo=datetime.timezone.utc, + ), + authority=PYPI_AUTHORITY, + fetcher=FETCHER, + format="pypi-project-json", + metadata=json.dumps(extrinsic_metadata).encode(), + origin=origin_url, + ), + ] + ), + call.raw_extrinsic_metadata_add( + [ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=parse_swhid( + "swh:1:rev:000007617b53e7b1458f695dd07de4ce55af1517" + ), + discovery_date=datetime.datetime( + 2019, 11, 11, 6, 21, 20, tzinfo=datetime.timezone.utc, + ), + authority=SWH_AUTHORITY, + fetcher=FETCHER, + format="original-artifact-json", + metadata=json.dumps(original_artifact).encode(), + origin=origin_url, + ), + ] + ), + ]