diff --git a/swh/loader/package/puppet/loader.py b/swh/loader/package/puppet/loader.py index 6f0221e..a171980 100644 --- a/swh/loader/package/puppet/loader.py +++ b/swh/loader/package/puppet/loader.py @@ -1,153 +1,153 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime import json from pathlib import Path -from typing import Any, Dict, Iterator, Optional, Sequence, Tuple +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple import attr import iso8601 from packaging.version import parse as parse_version from swh.loader.package.loader import BasePackageInfo, PackageLoader from swh.loader.package.utils import Person, release_name from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface @attr.s class PuppetPackageInfo(BasePackageInfo): name = attr.ib(type=str) """Name of the package""" filename = attr.ib(type=str) """Archive (tar.gz) file name""" version = attr.ib(type=str) """Current version""" last_modified = attr.ib(type=datetime) """Module last update date as release date""" def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: """Extract intrinsic metadata from metadata.json file at dir_path. Each Puppet module version has a metadata.json file at the root of the archive. See ``https://puppet.com/docs/puppet/7/modules_metadata.html`` for metadata specifications. Args: dir_path: A directory on disk where a metadata.json file must be present Returns: A dict mapping from json parser """ meta_json_path = dir_path / "metadata.json" metadata: Dict[str, Any] = json.loads(meta_json_path.read_text()) return metadata class PuppetLoader(PackageLoader[PuppetPackageInfo]): visit_type = "puppet" def __init__( self, storage: StorageInterface, url: str, - artifacts: Dict[str, Any], + artifacts: List[Dict[str, Any]], **kwargs, ): super().__init__(storage=storage, url=url, **kwargs) self.url = url - self.artifacts = artifacts + self.artifacts: Dict[str, Dict] = { + artifact["version"]: artifact for artifact in artifacts + } def get_versions(self) -> Sequence[str]: """Get all released versions of a Puppet module Returns: A sequence of versions Example:: ["0.1.1", "0.10.2"] """ versions = list(self.artifacts.keys()) versions.sort(key=parse_version) return versions def get_default_version(self) -> str: """Get the newest release version of a Puppet module Returns: A string representing a version Example:: "0.10.2" """ return self.get_versions()[-1] def get_package_info(self, version: str) -> Iterator[Tuple[str, PuppetPackageInfo]]: """Get release name and package information from version Args: version: Package version (e.g: "0.1.0") Returns: Iterator of tuple (release_name, p_info) """ data = self.artifacts[version] assert data["filename"].endswith(f"-{version}.tar.gz") pkgname: str = data["filename"].split(f"-{version}.tar.gz")[0] url: str = data["url"] filename: str = data["filename"] last_modified: datetime = iso8601.parse_date(data["last_update"]) p_info = PuppetPackageInfo( name=pkgname, filename=filename, url=url, version=version, last_modified=last_modified, checksums=data["checksums"], ) yield release_name(version), p_info def build_release( self, p_info: PuppetPackageInfo, uncompressed_path: str, directory: Sha1Git ) -> Optional[Release]: # compute extracted module directory name dirname = p_info.filename.split(".tar.gz")[0] # Extract intrinsic metadata from uncompressed_path/{dirname}/metadata.json intrinsic_metadata = extract_intrinsic_metadata( Path(uncompressed_path) / f"{dirname}" ) version: str = intrinsic_metadata["version"] assert version == p_info.version - description = intrinsic_metadata["summary"] author = Person.from_fullname(intrinsic_metadata["author"].encode()) message = ( f"Synthetic release for Puppet source package {p_info.name} " - f"version {version}\n\n" - f"{description}\n" + f"version {version}\n" ) return Release( name=version.encode(), author=author, date=TimestampWithTimezone.from_datetime(p_info.last_modified), message=message.encode(), target_type=ObjectType.DIRECTORY, target=directory, synthetic=True, ) diff --git a/swh/loader/package/puppet/tests/test_puppet.py b/swh/loader/package/puppet/tests/test_puppet.py index 07cf0ac..cc78d4d 100644 --- a/swh/loader/package/puppet/tests/test_puppet.py +++ b/swh/loader/package/puppet/tests/test_puppet.py @@ -1,125 +1,124 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.loader.package.puppet.loader import PuppetLoader from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes from swh.model.model import ( ObjectType, Person, Release, Snapshot, SnapshotBranch, TargetType, TimestampWithTimezone, ) ORIGINS = { "url": "https://forge.puppet.com/modules/saz/memcached", - "artifacts": { - "1.0.0": { + "artifacts": [ + { "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-1.0.0.tar.gz", # noqa: B950 "version": "1.0.0", "filename": "saz-memcached-1.0.0.tar.gz", "last_update": "2011-11-20T13:40:30-08:00", "checksums": { "length": 763, }, }, - "8.1.0": { + { "url": "https://forgeapi.puppet.com/v3/files/saz-memcached-8.1.0.tar.gz", # noqa: B950 "version": "8.1.0", "filename": "saz-memcached-8.1.0.tar.gz", "last_update": "2022-07-11T03:34:55-07:00", "checksums": { "md5": "5313e8fff0af08d63681daf955e7a604", "sha256": "0dbb1470c64435700767e9887d0cf70203b1ae59445c401d5d200f2dabb3226e", # noqa: B950 }, }, - }, + ], } def test_get_versions(requests_mock_datadir, swh_storage): loader = PuppetLoader( swh_storage, url=ORIGINS["url"], artifacts=ORIGINS["artifacts"] ) assert loader.get_versions() == ["1.0.0", "8.1.0"] def test_get_default_version(requests_mock_datadir, swh_storage): loader = PuppetLoader( swh_storage, url=ORIGINS["url"], artifacts=ORIGINS["artifacts"] ) assert loader.get_default_version() == "8.1.0" def test_puppet_loader_load_multiple_version( datadir, requests_mock_datadir, swh_storage ): loader = PuppetLoader( swh_storage, url=ORIGINS["url"], artifacts=ORIGINS["artifacts"] ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None - expected_snapshot_id = "9a8e76a8a6eae5285059d9f6d5083a99317727cf" + expected_snapshot_id = "c3da002f1dc325be29004fa64312f71ba50b9fbc" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"HEAD": SnapshotBranch( target=b"releases/8.1.0", target_type=TargetType.ALIAS, ), b"releases/1.0.0": SnapshotBranch( - target=hash_to_bytes("50eb560bb5322cd149359b9cc8debc78834bcfad"), + target=hash_to_bytes("83b3463dd35d44dbae4bfe917a9b127924a14bbd"), target_type=TargetType.RELEASE, ), b"releases/8.1.0": SnapshotBranch( - target=hash_to_bytes("2f5722136d775dd48fe85fabdd274f1e2d7fcf22"), + target=hash_to_bytes("90592c01fe7f96f32a88bc611193b305cb77cc03"), target_type=TargetType.RELEASE, ), }, ) check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert { "content": 1 + 1, "directory": 2 + 2, "origin": 1, "origin_visit": 1, "release": 1 + 1, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats assert swh_storage.release_get( - [hash_to_bytes("2f5722136d775dd48fe85fabdd274f1e2d7fcf22")] + [hash_to_bytes("90592c01fe7f96f32a88bc611193b305cb77cc03")] )[0] == Release( name=b"8.1.0", - message=b"Synthetic release for Puppet source package saz-memcached version 8.1.0\n\n" - b"Manage memcached via Puppet\n", + message=b"Synthetic release for Puppet source package saz-memcached version 8.1.0\n", target=hash_to_bytes("1b9a2dbc80f954e1ba4b2f1c6344d1ce4e84ab7c"), target_type=ObjectType.DIRECTORY, synthetic=True, author=Person(fullname=b"saz", name=b"saz", email=None), date=TimestampWithTimezone.from_iso8601("2022-07-11T03:34:55-07:00"), - id=hash_to_bytes("2f5722136d775dd48fe85fabdd274f1e2d7fcf22"), + id=hash_to_bytes("90592c01fe7f96f32a88bc611193b305cb77cc03"), ) assert_last_visit_matches( swh_storage, url=ORIGINS["url"], status="full", type="puppet", snapshot=expected_snapshot.id, ) diff --git a/swh/loader/package/puppet/tests/test_tasks.py b/swh/loader/package/puppet/tests/test_tasks.py index e3a0d5a..621e27c 100644 --- a/swh/loader/package/puppet/tests/test_tasks.py +++ b/swh/loader/package/puppet/tests/test_tasks.py @@ -1,50 +1,50 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister NAMESPACE = "swh.loader.package.puppet" @pytest.fixture def puppet_lister(): return Lister(name="puppet", instance_name="example", id=uuid.uuid4()) @pytest.fixture def puppet_listed_origin(puppet_lister): return ListedOrigin( lister_id=puppet_lister.id, url="some-url/api/packages/some-package", visit_type="aur", extra_loader_arguments={ - "artifacts": { - "1.0.0": { + "artifacts": [ + { "url": "https://domain/some-package-1.0.0.tar.gz", "version": "1.0.0", "filename": "some-module-1.0.0.tar.gz", "last_update": "2011-11-20T13:40:30-08:00", }, - } + ] }, ) def test_puppet_loader_task_for_listed_origin( loading_task_creation_for_listed_origin_test, puppet_lister, puppet_listed_origin, ): loading_task_creation_for_listed_origin_test( loader_class_name=f"{NAMESPACE}.loader.PuppetLoader", task_function_name=f"{NAMESPACE}.tasks.LoadPuppet", lister=puppet_lister, listed_origin=puppet_listed_origin, )