diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py --- a/swh/loader/package/opam/loader.py +++ b/swh/loader/package/opam/loader.py @@ -10,9 +10,20 @@ import attr -from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.loader.package.loader import ( + BasePackageInfo, + PackageLoader, + RawExtrinsicMetadataCore, +) from swh.loader.package.utils import cached_method -from swh.model.model import Person, Revision, RevisionType, Sha1Git +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + Person, + Revision, + RevisionType, + Sha1Git, +) from swh.storage.interface import StorageInterface @@ -103,6 +114,9 @@ def get_package_file(self, version: str) -> str: return f"{self.get_package_dir()}/{self.get_package_name(version)}/opam" + def get_metadata_authority(self): + return MetadataAuthority(type=MetadataAuthorityType.FORGE, url=self.opam_url) + @cached_method def _compute_versions(self) -> List[str]: """Compute the versions using opam internals @@ -171,24 +185,24 @@ """Return the most recent version of the package as default.""" return self._compute_versions()[-1] - def get_enclosed_single_line_field(self, field, version) -> Optional[str]: + def _opam_show_args(self, version: str): package_file = self.get_package_file(version) - result = opam_read( - [ - "opam", - "show", - "--color", - "never", - "--safe", - "--normalise", - "--root", - self.opam_root, - "--file", - package_file, - "--field", - field, - ] - ) + + return [ + "opam", + "show", + "--color", + "never", + "--safe", + "--normalise", + "--root", + self.opam_root, + "--file", + package_file, + ] + + def get_enclosed_single_line_field(self, field, version) -> Optional[str]: + result = opam_read(self._opam_show_args(version) + ["--field", field]) # Sanitize the result if any (remove trailing \n and enclosing ") return result.strip().strip('"') if result else None @@ -210,8 +224,21 @@ fullname = b"" if maintainer_field is None else str.encode(maintainer_field) committer = Person(fullname=fullname, name=None, email=None) + with Popen(self._opam_show_args(version) + ["--raw"], stdout=PIPE) as proc: + assert proc.stdout is not None + metadata = proc.stdout.read() + yield self.get_package_name(version), OpamPackageInfo( - url=url, filename=None, author=author, committer=committer, version=version + url=url, + filename=None, + author=author, + committer=committer, + version=version, + directory_extrinsic_metadata=[ + RawExtrinsicMetadataCore( + metadata=metadata, format="opam-package-definition", + ) + ], ) def build_revision( diff --git a/swh/loader/package/opam/tests/test_opam.py b/swh/loader/package/opam/tests/test_opam.py --- a/swh/loader/package/opam/tests/test_opam.py +++ b/swh/loader/package/opam/tests/test_opam.py @@ -3,11 +3,65 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - +from swh.loader.package import __version__ +from swh.loader.package.loader import RawExtrinsicMetadataCore from swh.loader.package.opam.loader import OpamLoader, OpamPackageInfo from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes -from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType +from swh.model.model import ( + MetadataAuthority, + MetadataAuthorityType, + MetadataFetcher, + Person, + RawExtrinsicMetadata, + Snapshot, + SnapshotBranch, + TargetType, +) +from swh.model.swhids import CoreSWHID, ExtendedObjectType, ExtendedSWHID, ObjectType +from swh.storage.interface import PagedResult + +OCB_METADATA = b"""\ +opam-version: "2.0" +name: "ocb" +version: "0.1" +synopsis: "SVG badge generator" +description: + "An OCaml library for SVG badge generation. There\'s also a command-line tool provided." +maintainer: "OCamlPro " +authors: "OCamlPro " +license: "ISC" +homepage: "https://ocamlpro.github.io/ocb/" +doc: "https://ocamlpro.github.io/ocb/api/" +bug-reports: "https://github.com/OCamlPro/ocb/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.0"} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/OCamlPro/ocb.git" +url { + src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz" + checksum: [ + "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46" + "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b" + ] +} +""" # noqa def test_opam_loader_no_opam_repository_fails(swh_storage, tmpdir, datadir): @@ -181,7 +235,6 @@ expected_package_info = OpamPackageInfo( url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz", filename=None, - directory_extrinsic_metadata=[], author=Person( fullname=b"OCamlPro ", name=None, email=None ), @@ -189,6 +242,11 @@ fullname=b"OCamlPro ", name=None, email=None ), version="0.1", + directory_extrinsic_metadata=[ + RawExtrinsicMetadataCore( + metadata=OCB_METADATA, format="opam-package-definition", + ) + ], ) assert branch_name == expected_branch_name @@ -202,3 +260,58 @@ assert revision.author == expected_package_info.author assert revision.committer == expected_package_info.committer + + +def test_opam_metadata(tmpdir, requests_mock_datadir, swh_storage, datadir): + opam_url = f"file://{datadir}/fake_opam_repo" + opam_root = tmpdir + opam_instance = "loadertest" + + opam_package = "ocb" + url = f"opam+{opam_url}/packages/{opam_package}" + + loader = OpamLoader( + swh_storage, + url, + opam_root, + opam_instance, + opam_url, + opam_package, + initialize_opam_root=True, + ) + + actual_load_status = loader.load() + + assert actual_load_status["status"] == "eventful" + + expected_revision_id = b"o\xad\x7f=\x07\xbb\xaah\xdbI(\xb0'\x10z\xfc\xff\x06x\x1b" + + revision = swh_storage.revision_get([expected_revision_id])[0] + assert revision is not None + + revision_swhid = CoreSWHID( + object_type=ObjectType.REVISION, object_id=expected_revision_id + ) + directory_swhid = ExtendedSWHID( + object_type=ExtendedObjectType.DIRECTORY, object_id=revision.directory + ) + metadata_authority = MetadataAuthority( + type=MetadataAuthorityType.FORGE, url=opam_url, + ) + expected_metadata = [ + RawExtrinsicMetadata( + target=directory_swhid, + authority=metadata_authority, + fetcher=MetadataFetcher( + name="swh.loader.package.opam.loader.OpamLoader", version=__version__, + ), + discovery_date=loader.visit_date, + format="opam-package-definition", + metadata=OCB_METADATA, + origin=url, + revision=revision_swhid, + ) + ] + assert swh_storage.raw_extrinsic_metadata_get( + directory_swhid, metadata_authority, + ) == PagedResult(next_page_token=None, results=expected_metadata,)