diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py --- a/swh/loader/package/opam/loader.py +++ b/swh/loader/package/opam/loader.py @@ -10,9 +10,20 @@ import attr -from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.loader.package.loader import ( + BasePackageInfo, + PackageLoader, + RawExtrinsicMetadataCore, +) from swh.loader.package.utils import cached_method -from swh.model.model import Person, Revision, RevisionType, Sha1Git +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + Person, + Revision, + RevisionType, + Sha1Git, +) from swh.storage.interface import StorageInterface @@ -103,6 +114,9 @@ def get_package_file(self, version: str) -> str: return f"{self.get_package_dir()}/{self.get_package_name(version)}/opam" + def get_metadata_authority(self): + return MetadataAuthority(type=MetadataAuthorityType.FORGE, url=self.opam_url) + @cached_method def _compute_versions(self) -> List[str]: """Compute the versions using opam internals @@ -171,9 +185,10 @@ """Return the most recent version of the package as default.""" return self._compute_versions()[-1] - def get_enclosed_single_line_field(self, field, version) -> Optional[str]: + def _opam_show_version_file(self, version: str, extra_params: List[str] = []): package_file = self.get_package_file(version) - result = opam_read( + + return opam_read( [ "opam", "show", @@ -185,11 +200,13 @@ self.opam_root, "--file", package_file, - "--field", - field, + *extra_params, ] ) + def get_enclosed_single_line_field(self, field, version) -> Optional[str]: + result = self._opam_show_version_file(version, ["--field", field]) + # Sanitize the result if any (remove trailing \n and enclosing ") return result.strip().strip('"') if result else None @@ -211,7 +228,17 @@ committer = Person(fullname=fullname, name=None, email=None) yield self.get_package_name(version), OpamPackageInfo( - url=url, filename=None, author=author, committer=committer, version=version + url=url, + filename=None, + author=author, + committer=committer, + version=version, + directory_extrinsic_metadata=[ + RawExtrinsicMetadataCore( + metadata=self._opam_show_version_file(version).encode(), + format="opam-package-definition", + ) + ], ) def build_revision( diff --git a/swh/loader/package/opam/tests/test_opam.py b/swh/loader/package/opam/tests/test_opam.py --- a/swh/loader/package/opam/tests/test_opam.py +++ b/swh/loader/package/opam/tests/test_opam.py @@ -4,10 +4,23 @@ # See top-level LICENSE file for more information +from swh.loader.package import __version__ +from swh.loader.package.loader import RawExtrinsicMetadataCore from swh.loader.package.opam.loader import OpamLoader, OpamPackageInfo from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes -from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + MetadataFetcher, + Person, + RawExtrinsicMetadata, + Snapshot, + SnapshotBranch, + TargetType, +) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType +from swh.storage.interface import PagedResult def test_opam_loader_no_opam_repository_fails(swh_storage, tmpdir, datadir): @@ -181,7 +194,6 @@ expected_package_info = OpamPackageInfo( url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz", filename=None, - directory_extrinsic_metadata=[], author=Person( fullname=b"OCamlPro ", name=None, email=None ), @@ -189,6 +201,11 @@ fullname=b"OCamlPro ", name=None, email=None ), version="0.1", + directory_extrinsic_metadata=[ + RawExtrinsicMetadataCore( + metadata=b'opam-version: "2.0"\n', format="opam-package-definition", + ) + ], ) assert branch_name == expected_branch_name @@ -202,3 +219,58 @@ assert revision.author == expected_package_info.author assert revision.committer == expected_package_info.committer + + +def test_opam_metadata(tmpdir, requests_mock_datadir, swh_storage, datadir): + opam_url = f"file://{datadir}/fake_opam_repo" + opam_root = tmpdir + opam_instance = "loadertest" + + opam_package = "ocb" + url = f"opam+{opam_url}/packages/{opam_package}" + + loader = OpamLoader( + swh_storage, + url, + opam_root, + opam_instance, + opam_url, + opam_package, + initialize_opam_root=True, + ) + + actual_load_status = loader.load() + + assert actual_load_status["status"] == "eventful" + + expected_revision_id = b"o\xad\x7f=\x07\xbb\xaah\xdbI(\xb0'\x10z\xfc\xff\x06x\x1b" + + revision = swh_storage.revision_get([expected_revision_id])[0] + assert revision is not None + + revision_swhid = CoreSWHID( + object_type=ObjectType.REVISION, object_id=expected_revision_id + ) + directory_swhid = ExtendedSWHID( + object_type=ExtendedObjectType.DIRECTORY, object_id=revision.directory + ) + metadata_authority = MetadataAuthority( + type=MetadataAuthorityType.FORGE, url=opam_url, + ) + expected_metadata = [ + RawExtrinsicMetadata( + target=directory_swhid, + authority=metadata_authority, + fetcher=MetadataFetcher( + name="swh.loader.package.opam.loader.OpamLoader", version=__version__, + ), + discovery_date=loader.visit_date, + format="opam-package-definition", + metadata=b'opam-version: "2.0"\n', + origin=url, + revision=revision_swhid, + ) + ] + assert swh_storage.raw_extrinsic_metadata_get( + directory_swhid, metadata_authority, + ) == PagedResult(next_page_token=None, results=expected_metadata,)