diff --git a/docs/package-loader-specifications.rst b/docs/package-loader-specifications.rst --- a/docs/package-loader-specifications.rst +++ b/docs/package-loader-specifications.rst @@ -144,11 +144,11 @@ - ``p_info.​version`` - ``release_name(​version)`` - =version - - Synthetic release for pub.dev source package {name} version {version} {description} + - Synthetic release for pub.dev source package {p_info.name} version {p_info.version} - true - from extrinsic metadata - from extrinsic metadata - - name, version and description from intrinsic metadata + - name and version from extrinsic metadata * - puppet - ``p_info.​version`` - ``release_name(​version)`` diff --git a/swh/loader/package/pubdev/loader.py b/swh/loader/package/pubdev/loader.py --- a/swh/loader/package/pubdev/loader.py +++ b/swh/loader/package/pubdev/loader.py @@ -3,12 +3,10 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json -from pathlib import Path -from typing import Any, Dict, Iterator, Optional, Sequence, Tuple +from typing import Dict, Iterator, Optional, Sequence, Tuple import attr from packaging.version import parse as parse_version -import yaml from swh.loader.package.loader import BasePackageInfo, PackageLoader from swh.loader.package.utils import ( @@ -37,26 +35,6 @@ author = attr.ib(type=Person) """Author""" - description = attr.ib(type=str) - """Description""" - - -def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: - """Extract intrinsic metadata from pubspec.yaml file at dir_path. - - Each pub.dev package version has a pubspec.yaml file at the root of the archive. - - See https://dart.dev/tools/pub/pubspec for pubspec specifications. - - Args: - dir_path: A directory on disk where a pubspec.yaml must be present - - Returns: - A dict mapping from yaml parser - """ - pubspec_path = dir_path / "pubspec.yaml" - return yaml.safe_load(pubspec_path.read_text()) - class PubDevLoader(PackageLoader[PubDevPackageInfo]): visit_type = "pubdev" @@ -77,14 +55,11 @@ self.PUBDEV_BASE_URL, f"{self.PUBDEV_BASE_URL}api/" ) - def _raw_info(self) -> bytes: - return get_url_body(self.package_info_url) - @cached_method def info(self) -> Dict: """Return the project metadata information (fetched from pub.dev registry)""" # Use strict=False in order to correctly manage case where \n is present in a string - info = json.loads(self._raw_info(), strict=False) + info = json.loads(get_url_body(self.package_info_url), strict=False) # Arrange versions list as a new dict with `version` as key versions = {v["version"]: v for v in info["versions"]} info["versions"] = versions @@ -135,17 +110,17 @@ name = v["pubspec"]["name"] filename = f"{name}-{version}.tar.gz" last_modified = v["published"] + checksums = {"sha256": v["archive_sha256"]} if v.get("archive_sha256") else {} - if "authors" in v["pubspec"]: + authors = v.get("pubspec", {}).get("authors") + if authors and isinstance(authors, list): # TODO: here we have a list of author, see T3887 - author = Person.from_fullname(v["pubspec"]["authors"][0].encode()) - elif "author" in v["pubspec"] and v["pubspec"]["author"] is not None: + author = Person.from_fullname(authors[0].encode()) + elif v.get("pubspec", {}).get("author"): author = Person.from_fullname(v["pubspec"]["author"].encode()) else: author = EMPTY_AUTHOR - description = v["pubspec"]["description"] - p_info = PubDevPackageInfo( name=name, filename=filename, @@ -153,8 +128,7 @@ version=version, last_modified=last_modified, author=author, - description=description, - checksums={"sha256": v["archive_sha256"]}, + checksums=checksums, ) yield release_name(version), p_info @@ -162,31 +136,14 @@ self, p_info: PubDevPackageInfo, uncompressed_path: str, directory: Sha1Git ) -> Optional[Release]: - # Extract intrinsic metadata from uncompressed_path/pubspec.yaml - intrinsic_metadata = extract_intrinsic_metadata(Path(uncompressed_path)) - - name: str = intrinsic_metadata["name"] - version: str = intrinsic_metadata["version"] - assert version == p_info.version - - # author from intrinsic_metadata should not take precedence over the one - # returned by the api, see https://dart.dev/tools/pub/pubspec#authorauthors - author: Person = p_info.author - - if "description" in intrinsic_metadata and intrinsic_metadata["description"]: - description = intrinsic_metadata["description"] - else: - description = p_info.description - message = ( - f"Synthetic release for pub.dev source package {name} " - f"version {version}\n\n" - f"{description}\n" + f"Synthetic release for pub.dev source package {p_info.name} " + f"version {p_info.version}\n" ) return Release( - name=version.encode(), - author=author, + name=p_info.version.encode(), + author=p_info.author, date=TimestampWithTimezone.from_iso8601(p_info.last_modified), message=message.encode(), target_type=ObjectType.DIRECTORY, diff --git a/swh/loader/package/pubdev/tests/test_pubdev.py b/swh/loader/package/pubdev/tests/test_pubdev.py --- a/swh/loader/package/pubdev/tests/test_pubdev.py +++ b/swh/loader/package/pubdev/tests/test_pubdev.py @@ -129,8 +129,8 @@ assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None - expected_snapshot_id = "245092931ba809e6c54ebda8f865fb5a969a4134" - expected_release_id = "919f267ea050539606344d49d14bf594c4386e5a" + expected_snapshot_id = "dffca49aec93fcf1fa63fa25bf9a04c833a30d73" + expected_release_id = "1e2e7226ac9136f2eb7ce28f32ca08fff28590b1" assert expected_snapshot_id == load_status["snapshot_id"] @@ -164,8 +164,7 @@ assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( name=b"0.1.1", - message=b"Synthetic release for pub.dev source package Autolinker version" - b" 0.1.1\n\nPort of Autolinker.js to dart\n", + message=b"Synthetic release for pub.dev source package Autolinker version 0.1.1\n", target=hash_to_bytes("3fb6d4f2c0334d1604357ae92b2dd38a55a78194"), target_type=ObjectType.DIRECTORY, synthetic=True, @@ -199,7 +198,7 @@ assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None - expected_snapshot_id = "43d5b68a9fa973aa95e56916aaef70841ccbc2a0" + expected_snapshot_id = "b03a4ef56b1a3bd4812f8e37f439c261cf4fd2c7" assert expected_snapshot_id == load_status["snapshot_id"] @@ -207,11 +206,11 @@ id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/1.0.0": SnapshotBranch( - target=hash_to_bytes("fbf8e40af675096681954553d737861e10b57216"), + target=hash_to_bytes("6f6eecd1ced321778d6a4bc60af4fb0e93178307"), target_type=TargetType.RELEASE, ), b"releases/3.8.2": SnapshotBranch( - target=hash_to_bytes("627a5d586e3fb4e7319b17f1aee268fe2fb8e01c"), + target=hash_to_bytes("012bac381e2b9cda7de2da0391bc2969bf80ff97"), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( @@ -253,8 +252,8 @@ assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None - expected_snapshot_id = "4fa9f19d1d6ccc70921c8c50b278f510db63aa36" - expected_release_id = "538c98fd69a42d8d0561a7ca95b354de2143a3ab" + expected_snapshot_id = "2af571a302514bf17807dc114fff15501f8c1387" + expected_release_id = "87331a7804673cb00a339b504d2345769b7ae34a" assert expected_snapshot_id == load_status["snapshot_id"] @@ -292,8 +291,8 @@ assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None - expected_snapshot_id = "0c7fa6b9fced23c648d2093ad5597622683f8aed" - expected_release_id = "7d8c05181069aa1049a3f0bc1d13bedc34625d47" + expected_snapshot_id = "8b86c9fb49bbf3e2b4513dc35a2838c67e8895bc" + expected_release_id = "d6ba845e28fba2a51e2ed358664cad645a2591ca" assert expected_snapshot_id == load_status["snapshot_id"]