Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/migrate_extrinsic_metadata/test_npm.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
# flake8: noqa | # flake8: noqa | ||||
# because of long lines | # because of long lines | ||||
import copy | import copy | ||||
import datetime | import datetime | ||||
import json | import json | ||||
from unittest.mock import Mock, call | from unittest.mock import Mock, call | ||||
from swh.model.identifiers import parse_swhid | |||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | |||||
Origin, | Origin, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
) | ) | ||||
from swh.model.swhid import SWHID, SWHIDObjectType, parse_swhid | |||||
from swh.storage.migrate_extrinsic_metadata import ( | from swh.storage.migrate_extrinsic_metadata import ( | ||||
handle_row, | handle_row, | ||||
npm_package_from_source_url, | npm_package_from_source_url, | ||||
) | ) | ||||
FETCHER = MetadataFetcher( | FETCHER = MetadataFetcher( | ||||
name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", | name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 90 Lines • ▼ Show 20 Lines | row = { | ||||
"raw": {"name": "@l3ilkojr/jdinsults", "version": "3.0.0"}, | "raw": {"name": "@l3ilkojr/jdinsults", "version": "3.0.0"}, | ||||
"tool": "package.json", | "tool": "package.json", | ||||
}, | }, | ||||
"original_artifact": original_artifacts, | "original_artifact": original_artifacts, | ||||
}, | }, | ||||
} | } | ||||
origin_url = "https://www.npmjs.com/package/@l3ilkojr/jdinsults" | origin_url = "https://www.npmjs.com/package/@l3ilkojr/jdinsults" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, | 2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=NPM_AUTHORITY, | authority=NPM_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="replicate-npm-package-json", | format="replicate-npm-package-json", | ||||
metadata=json.dumps(extrinsic_metadata).encode(), | metadata=json.dumps(extrinsic_metadata).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" | "swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, | 2020, 2, 27, 1, 35, 47, 965375, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(original_artifacts).encode(), | metadata=json.dumps(original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" | "swh:1:rev:000002a49bba17ca8cf37f5f3d16aaacf95360fc" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | row = { | ||||
2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc | 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc | ||||
), | ), | ||||
"type": "tar", | "type": "tar", | ||||
"message": b"0.0.6", | "message": b"0.0.6", | ||||
"metadata": {"package": extrinsic_metadata, "package_source": package_source,}, | "metadata": {"package": extrinsic_metadata, "package_source": package_source,}, | ||||
} | } | ||||
origin_url = "https://www.npmjs.com/package/simplemaps" | origin_url = "https://www.npmjs.com/package/simplemaps" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, | 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=NPM_AUTHORITY, | authority=NPM_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="replicate-npm-package-json", | format="replicate-npm-package-json", | ||||
metadata=json.dumps(extrinsic_metadata).encode(), | metadata=json.dumps(extrinsic_metadata).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" | "swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, | 2016, 12, 23, 7, 21, 29, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(original_artifacts).encode(), | metadata=json.dumps(original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" | "swh:1:rev:000004aeed09ee089c781264c04d2564fd58feb5" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] | ||||
Show All 40 Lines | row = { | ||||
2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc | 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc | ||||
), | ), | ||||
"type": "tar", | "type": "tar", | ||||
"message": b"0.1.11", | "message": b"0.1.11", | ||||
"metadata": {"package": extrinsic_metadata, "package_source": package_source,}, | "metadata": {"package": extrinsic_metadata, "package_source": package_source,}, | ||||
} | } | ||||
origin_url = "https://www.npmjs.com/package/@piximi/components" | origin_url = "https://www.npmjs.com/package/@piximi/components" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, | 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=NPM_AUTHORITY, | authority=NPM_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="replicate-npm-package-json", | format="replicate-npm-package-json", | ||||
metadata=json.dumps(extrinsic_metadata).encode(), | metadata=json.dumps(extrinsic_metadata).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" | "swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, | 2019, 6, 7, 19, 56, 4, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(original_artifacts).encode(), | metadata=json.dumps(original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" | "swh:1:rev:00002019c5775874bced007a529bd3b78bf60457" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] |