Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/migrate_extrinsic_metadata/test_cran.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
# flake8: noqa | # flake8: noqa | ||||
# because of long lines | # because of long lines | ||||
import copy | import copy | ||||
import datetime | import datetime | ||||
import json | import json | ||||
from unittest.mock import Mock, call | from unittest.mock import Mock, call | ||||
from swh.model.identifiers import parse_swhid | |||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | |||||
Origin, | Origin, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
) | ) | ||||
from swh.model.swhid import SWHID, SWHIDObjectType, parse_swhid | |||||
from swh.storage.migrate_extrinsic_metadata import cran_package_from_url, handle_row | from swh.storage.migrate_extrinsic_metadata import cran_package_from_url, handle_row | ||||
FETCHER = MetadataFetcher( | FETCHER = MetadataFetcher( | ||||
name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", | name="migrate-extrinsic-metadata-from-revisions", version="0.0.1", | ||||
) | ) | ||||
SWH_AUTHORITY = MetadataAuthority( | SWH_AUTHORITY = MetadataAuthority( | ||||
type=MetadataAuthorityType.REGISTRY, | type=MetadataAuthorityType.REGISTRY, | ||||
url="https://softwareheritage.org/", | url="https://softwareheritage.org/", | ||||
▲ Show 20 Lines • Show All 66 Lines • ▼ Show 20 Lines | row = { | ||||
}, | }, | ||||
"tool": "DESCRIPTION", | "tool": "DESCRIPTION", | ||||
}, | }, | ||||
"original_artifact": source_original_artifacts, | "original_artifact": source_original_artifacts, | ||||
}, | }, | ||||
} | } | ||||
origin_url = "https://cran.r-project.org/package=ExtremeRisks" | origin_url = "https://cran.r-project.org/package=ExtremeRisks" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(row, storage, deposit_cur, dry_run=False) | handle_row(row, storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2020, 5, 7, 15, 27, 38, 652281, tzinfo=datetime.timezone.utc, | 2020, 5, 7, 15, 27, 38, 652281, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(dest_original_artifacts).encode(), | metadata=json.dumps(dest_original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:000361aa33842cbdea5fa6e77db696b937ebd269" | "swh:1:rev:000361aa33842cbdea5fa6e77db696b937ebd269" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | row = { | ||||
}, | }, | ||||
"tool": "DESCRIPTION", | "tool": "DESCRIPTION", | ||||
}, | }, | ||||
"original_artifact": source_original_artifacts, | "original_artifact": source_original_artifacts, | ||||
}, | }, | ||||
} | } | ||||
origin_url = "https://cran.r-project.org/package=gofgamma" | origin_url = "https://cran.r-project.org/package=gofgamma" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | handle_row(copy.deepcopy(row), storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2020, 4, 30, 11, 1, 57, 832481, tzinfo=datetime.timezone.utc, | 2020, 4, 30, 11, 1, 57, 832481, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(dest_original_artifacts).encode(), | metadata=json.dumps(dest_original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:0000d4ef5e166122aee6862ad38a18ce5386cc3e" | "swh:1:rev:0000d4ef5e166122aee6862ad38a18ce5386cc3e" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] | ||||
Show All 36 Lines | row = { | ||||
}, | }, | ||||
"tool": "DESCRIPTION", | "tool": "DESCRIPTION", | ||||
}, | }, | ||||
"original_artifact": original_artifacts, | "original_artifact": original_artifacts, | ||||
}, | }, | ||||
} | } | ||||
origin_url = "https://cran.r-project.org/package=r2mlm" | origin_url = "https://cran.r-project.org/package=r2mlm" | ||||
origin_swhid = SWHID( | |||||
object_type=SWHIDObjectType.ORIGIN, object_id=origin_url.encode() | |||||
) | |||||
storage = Mock() | storage = Mock() | ||||
def origin_get(urls): | def origin_get(urls): | ||||
assert urls == [origin_url] | assert urls == [origin_url] | ||||
return [Origin(url=origin_url)] | return [Origin(url=origin_url)] | ||||
storage.origin_get.side_effect = origin_get | storage.origin_get.side_effect = origin_get | ||||
deposit_cur = None | deposit_cur = None | ||||
handle_row(row, storage, deposit_cur, dry_run=False) | handle_row(row, storage, deposit_cur, dry_run=False) | ||||
assert storage.method_calls == [ | assert storage.method_calls == [ | ||||
call.origin_get([origin_url]), | call.origin_get([origin_url]), | ||||
call.raw_extrinsic_metadata_add( | call.raw_extrinsic_metadata_add( | ||||
[ | [ | ||||
RawExtrinsicMetadata( | RawExtrinsicMetadata( | ||||
type=MetadataTargetType.DIRECTORY, | |||||
target=DIRECTORY_SWHID, | target=DIRECTORY_SWHID, | ||||
discovery_date=datetime.datetime( | discovery_date=datetime.datetime( | ||||
2020, 9, 25, 14, 4, 20, 926667, tzinfo=datetime.timezone.utc, | 2020, 9, 25, 14, 4, 20, 926667, tzinfo=datetime.timezone.utc, | ||||
), | ), | ||||
authority=SWH_AUTHORITY, | authority=SWH_AUTHORITY, | ||||
fetcher=FETCHER, | fetcher=FETCHER, | ||||
format="original-artifacts-json", | format="original-artifacts-json", | ||||
metadata=json.dumps(original_artifacts).encode(), | metadata=json.dumps(original_artifacts).encode(), | ||||
origin=origin_url, | origin=origin_swhid, | ||||
revision=parse_swhid( | revision=parse_swhid( | ||||
"swh:1:rev:2e223782ee4ba152e4c886f797976241c39a9aab" | "swh:1:rev:2e223782ee4ba152e4c886f797976241c39a9aab" | ||||
), | ), | ||||
), | ), | ||||
] | ] | ||||
), | ), | ||||
] | ] |