diff --git a/swh/loader/package/npm/loader.py b/swh/loader/package/npm/loader.py --- a/swh/loader/package/npm/loader.py +++ b/swh/loader/package/npm/loader.py @@ -15,6 +15,8 @@ from urllib.parse import quote from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, Person, RevisionType, Revision, @@ -22,7 +24,11 @@ Sha1Git, ) -from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.loader.package.loader import ( + BasePackageInfo, + PackageLoader, + RawExtrinsicMetadataCore, +) from swh.loader.package.utils import api_info, release_name @@ -96,7 +102,8 @@ """ if not self._info: - self._info = json.loads(api_info(self.provider_url)) + self._raw_info = api_info(self.provider_url) + self._info = json.loads(self._raw_info) return self._info def get_versions(self) -> Sequence[str]: @@ -105,6 +112,22 @@ def get_default_version(self) -> str: return self.info["dist-tags"].get("latest", "") + def get_metadata_authority(self): + return MetadataAuthority( + type=MetadataAuthorityType.FORGE, + url="https://replicate.npmjs.com/", + metadata={}, + ) + + def get_extrinsic_snapshot_metadata(self): + return [ + RawExtrinsicMetadataCore( + format="replicate-npm-package-json", + metadata=self._raw_info, + discovery_date=None, + ), + ] + def get_package_info(self, version: str) -> Iterator[Tuple[str, NpmPackageInfo]]: p_info = NpmPackageInfo.from_metadata( project_metadata=self.info, version=version diff --git a/swh/loader/package/npm/tests/test_npm.py b/swh/loader/package/npm/tests/test_npm.py --- a/swh/loader/package/npm/tests/test_npm.py +++ b/swh/loader/package/npm/tests/test_npm.py @@ -7,9 +7,23 @@ import os import pytest -from swh.model.hashutil import hash_to_bytes -from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType +from swh.model.hashutil import hash_to_bytes, hash_to_hex +from swh.model.identifiers import SWHID +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + MetadataFetcher, + MetadataTargetType, + Person, + RawExtrinsicMetadata, + Snapshot, + SnapshotBranch, + TargetType, +) + +from swh.storage.interface import PagedResult +from swh.loader.package import __version__ from swh.loader.package.npm.loader import ( _author_str, NpmLoader, @@ -24,6 +38,12 @@ ) +@pytest.fixture +def org_api_info(datadir) -> bytes: + with open(os.path.join(datadir, "https_replicate.npmjs.com", "org"), "rb",) as f: + return f.read() + + def test_npm_author_str(): for author, expected_author in [ ("author", "author"), @@ -322,7 +342,7 @@ ) -def test_npm_loader_first_visit(swh_config, requests_mock_datadir): +def test_npm_loader_first_visit(swh_config, requests_mock_datadir, org_api_info): package = "org" url = package_url(package) loader = NpmLoader(url) @@ -387,6 +407,32 @@ ) check_snapshot(expected_snapshot, loader.storage) + snapshot_swhid = SWHID( + object_type="snapshot", object_id=hash_to_hex(expected_snapshot_id) + ) + metadata_authority = MetadataAuthority( + type=MetadataAuthorityType.FORGE, url="https://replicate.npmjs.com/", + ) + expected_metadata = [ + RawExtrinsicMetadata( + type=MetadataTargetType.SNAPSHOT, + id=snapshot_swhid, + authority=metadata_authority, + fetcher=MetadataFetcher( + name="swh.loader.package.npm.loader.NpmLoader", version=__version__, + ), + discovery_date=loader.visit_date, + format="replicate-npm-package-json", + metadata=org_api_info, + origin="https://www.npmjs.com/package/org", + ) + ] + assert loader.storage.raw_extrinsic_metadata_get( + type=MetadataTargetType.SNAPSHOT, + id=snapshot_swhid, + authority=metadata_authority, + ) == PagedResult(next_page_token=None, results=expected_metadata,) + def test_npm_loader_incremental_visit(swh_config, requests_mock_datadir_visits): package = "org"