diff --git a/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py --- a/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py +++ b/swh/storage/tests/migrate_extrinsic_metadata/test_pypi.py @@ -9,7 +9,8 @@ import copy import datetime import json -from unittest.mock import call, Mock + +import attr from swh.model.identifiers import parse_swhid from swh.model.model import ( @@ -21,6 +22,8 @@ RawExtrinsicMetadata, ) +from swh.storage import get_storage +from swh.storage.interface import PagedResult from swh.storage.migrate_extrinsic_metadata import ( handle_row, pypi_project_from_filename, @@ -31,12 +34,10 @@ name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", ) PYPI_AUTHORITY = MetadataAuthority( - type=MetadataAuthorityType.FORGE, url="https://pypi.org/", metadata={}, + type=MetadataAuthorityType.FORGE, url="https://pypi.org/", ) SWH_AUTHORITY = MetadataAuthority( - type=MetadataAuthorityType.REGISTRY, - url="https://softwareheritage.org/", - metadata={}, + type=MetadataAuthorityType.REGISTRY, url="https://softwareheritage.org/", ) @@ -119,55 +120,58 @@ origin_url = "https://pypi.org/project/m3-ui/" - storage = Mock() - - def origin_get(urls): - assert urls == [origin_url] - return [Origin(url=origin_url)] + storage = get_storage("memory") + storage.origin_add([Origin(url=origin_url)]) + storage.metadata_authority_add( + [ + attr.evolve(PYPI_AUTHORITY, metadata={}), + attr.evolve(SWH_AUTHORITY, metadata={}), + ] + ) + storage.metadata_fetcher_add([FETCHER]) - storage.origin_get.side_effect = origin_get deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) - assert storage.method_calls == [ - call.origin_get([origin_url]), - call.raw_extrinsic_metadata_add( - [ - RawExtrinsicMetadata( - type=MetadataTargetType.REVISION, - id=parse_swhid( - "swh:1:rev:000007617b53e7b1458f695dd07de4ce55af1517" - ), - discovery_date=datetime.datetime( - 2020, 1, 23, 18, 43, 9, 109407, tzinfo=datetime.timezone.utc, - ), - authority=PYPI_AUTHORITY, - fetcher=FETCHER, - format="pypi-project-json", - metadata=json.dumps(extrinsic_metadata).encode(), - origin=origin_url, + revision_swhid = parse_swhid("swh:1:rev:000007617b53e7b1458f695dd07de4ce55af1517") + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=PYPI_AUTHORITY, + ) == PagedResult( + results=[ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=revision_swhid, + discovery_date=datetime.datetime( + 2020, 1, 23, 18, 43, 9, 109407, tzinfo=datetime.timezone.utc, ), - ] - ), - call.raw_extrinsic_metadata_add( - [ - RawExtrinsicMetadata( - type=MetadataTargetType.REVISION, - id=parse_swhid( - "swh:1:rev:000007617b53e7b1458f695dd07de4ce55af1517" - ), - discovery_date=datetime.datetime( - 2020, 1, 23, 18, 43, 9, 109407, tzinfo=datetime.timezone.utc, - ), - authority=SWH_AUTHORITY, - fetcher=FETCHER, - format="original-artifacts-json", - metadata=json.dumps(original_artifacts).encode(), - origin=origin_url, + authority=PYPI_AUTHORITY, + fetcher=FETCHER, + format="pypi-project-json", + metadata=json.dumps(extrinsic_metadata).encode(), + origin=origin_url, + ), + ], + next_page_token=None, + ) + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=SWH_AUTHORITY, + ) == PagedResult( + results=[ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=revision_swhid, + discovery_date=datetime.datetime( + 2020, 1, 23, 18, 43, 9, 109407, tzinfo=datetime.timezone.utc, ), - ] - ), - ] + authority=SWH_AUTHORITY, + fetcher=FETCHER, + format="original-artifacts-json", + metadata=json.dumps(original_artifacts).encode(), + origin=origin_url, + ), + ], + next_page_token=None, + ) def test_pypi_2(): @@ -230,54 +234,59 @@ origin_url = "https://pypi.org/project/jupyterhub-simx/" - storage = Mock() - - def origin_get(urls): - assert urls == [origin_url] - return [Origin(url=origin_url)] + storage = get_storage("memory") - storage.origin_get.side_effect = origin_get + storage.origin_add([Origin(url=origin_url)]) + storage.metadata_authority_add( + [ + attr.evolve(PYPI_AUTHORITY, metadata={}), + attr.evolve(SWH_AUTHORITY, metadata={}), + ] + ) + storage.metadata_fetcher_add([FETCHER]) deposit_cur = None + handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) - assert storage.method_calls == [ - call.raw_extrinsic_metadata_add( - [ - RawExtrinsicMetadata( - type=MetadataTargetType.REVISION, - id=parse_swhid( - "swh:1:rev:000004d6382c4ad4c0519266626c36551f0e51ca" - ), - discovery_date=datetime.datetime( - 2019, 1, 23, 22, 10, 55, tzinfo=datetime.timezone.utc, - ), - authority=PYPI_AUTHORITY, - fetcher=FETCHER, - format="pypi-project-json", - metadata=json.dumps(extrinsic_metadata).encode(), - origin=None, + revision_swhid = parse_swhid("swh:1:rev:000004d6382c4ad4c0519266626c36551f0e51ca") + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=PYPI_AUTHORITY, + ) == PagedResult( + results=[ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=revision_swhid, + discovery_date=datetime.datetime( + 2019, 1, 23, 22, 10, 55, tzinfo=datetime.timezone.utc, ), - ] - ), - call.raw_extrinsic_metadata_add( - [ - RawExtrinsicMetadata( - type=MetadataTargetType.REVISION, - id=parse_swhid( - "swh:1:rev:000004d6382c4ad4c0519266626c36551f0e51ca" - ), - discovery_date=datetime.datetime( - 2019, 1, 23, 22, 10, 55, tzinfo=datetime.timezone.utc, - ), - authority=SWH_AUTHORITY, - fetcher=FETCHER, - format="original-artifacts-json", - metadata=json.dumps(dest_original_artifacts).encode(), - origin=None, + authority=PYPI_AUTHORITY, + fetcher=FETCHER, + format="pypi-project-json", + metadata=json.dumps(extrinsic_metadata).encode(), + origin=None, + ), + ], + next_page_token=None, + ) + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=SWH_AUTHORITY, + ) == PagedResult( + results=[ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=revision_swhid, + discovery_date=datetime.datetime( + 2019, 1, 23, 22, 10, 55, tzinfo=datetime.timezone.utc, ), - ] - ), - ] + authority=SWH_AUTHORITY, + fetcher=FETCHER, + format="original-artifacts-json", + metadata=json.dumps(dest_original_artifacts).encode(), + origin=None, + ), + ], + next_page_token=None, + ) def test_pypi_3(): @@ -324,33 +333,40 @@ origin_url = "https://pypi.org/project/PyPDFLite/" - storage = Mock() + storage = get_storage("memory") - def origin_get(urls): - assert urls == [origin_url] - return [Origin(url=origin_url)] - - storage.origin_get.side_effect = origin_get + storage.origin_add([Origin(url=origin_url)]) + storage.metadata_authority_add( + [ + attr.evolve(PYPI_AUTHORITY, metadata={}), + attr.evolve(SWH_AUTHORITY, metadata={}), + ] + ) + storage.metadata_fetcher_add([FETCHER]) deposit_cur = None handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) - assert storage.method_calls == [ - call.raw_extrinsic_metadata_add( - [ - RawExtrinsicMetadata( - type=MetadataTargetType.REVISION, - id=parse_swhid( - "swh:1:rev:4ea9917cdf53cd13534a042e4eb3787b86c834d2" - ), - discovery_date=datetime.datetime( - 2014, 5, 7, 22, 3, tzinfo=datetime.timezone.utc, - ), - authority=SWH_AUTHORITY, - fetcher=FETCHER, - format="original-artifacts-json", - metadata=json.dumps(dest_original_artifacts).encode(), - origin=None, + revision_swhid = parse_swhid("swh:1:rev:4ea9917cdf53cd13534a042e4eb3787b86c834d2") + + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=PYPI_AUTHORITY, + ) == PagedResult(results=[], next_page_token=None,) + assert storage.raw_extrinsic_metadata_get( + MetadataTargetType.REVISION, revision_swhid, authority=SWH_AUTHORITY, + ) == PagedResult( + results=[ + RawExtrinsicMetadata( + type=MetadataTargetType.REVISION, + id=revision_swhid, + discovery_date=datetime.datetime( + 2014, 5, 7, 22, 3, tzinfo=datetime.timezone.utc, ), - ] - ), - ] + authority=SWH_AUTHORITY, + fetcher=FETCHER, + format="original-artifacts-json", + metadata=json.dumps(dest_original_artifacts).encode(), + origin=None, + ), + ], + next_page_token=None, + )