diff --git a/swh/loader/package/pubdev/loader.py b/swh/loader/package/pubdev/loader.py index e278b0d..2a0a944 100644 --- a/swh/loader/package/pubdev/loader.py +++ b/swh/loader/package/pubdev/loader.py @@ -1,188 +1,194 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from distutils.version import LooseVersion import json from pathlib import Path from typing import Any, Dict, Iterator, Optional, Sequence, Tuple import attr import yaml from swh.loader.package.loader import BasePackageInfo, PackageLoader from swh.loader.package.utils import ( EMPTY_AUTHOR, Person, api_info, cached_method, release_name, ) from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface @attr.s class PubDevPackageInfo(BasePackageInfo): name = attr.ib(type=str) """Name of the package""" version = attr.ib(type=str) """Current version""" last_modified = attr.ib(type=str) """Last modified date as release date""" author = attr.ib(type=Person) """Author""" description = attr.ib(type=str) """Description""" def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: """Extract intrinsic metadata from pubspec.yaml file at dir_path. Each pub.dev package version has a pubspec.yaml file at the root of the archive. See https://dart.dev/tools/pub/pubspec for pubspec specifications. Args: dir_path: A directory on disk where a pubspec.yaml must be present Returns: A dict mapping from yaml parser """ pubspec_path = dir_path / "pubspec.yaml" return yaml.safe_load(pubspec_path.read_text()) class PubDevLoader(PackageLoader[PubDevPackageInfo]): visit_type = "pubdev" + PUBDEV_BASE_URL = "https://pub.dev/" + def __init__( self, storage: StorageInterface, url: str, **kwargs, ): super().__init__(storage=storage, url=url, **kwargs) self.url = url + assert url.startswith(self.PUBDEV_BASE_URL) + self.package_info_url = url.replace( + self.PUBDEV_BASE_URL, f"{self.PUBDEV_BASE_URL}api/" + ) def _raw_info(self) -> bytes: - return api_info(self.url) + return api_info(self.package_info_url) @cached_method def info(self) -> Dict: """Return the project metadata information (fetched from pub.dev registry)""" # Use strict=False in order to correctly manage case where \n is present in a string info = json.loads(self._raw_info(), strict=False) # Arrange versions list as a new dict with `version` as key versions = {v["version"]: v for v in info["versions"]} info["versions"] = versions return info def get_versions(self) -> Sequence[str]: """Get all released versions of a PubDev package Returns: A sequence of versions Example:: ["0.1.1", "0.10.2"] """ versions = list(self.info()["versions"].keys()) versions.sort(key=LooseVersion) return versions def get_default_version(self) -> str: """Get the newest release version of a PubDev package Returns: A string representing a version Example:: "0.1.2" """ latest = self.info()["latest"] return latest["version"] def get_package_info(self, version: str) -> Iterator[Tuple[str, PubDevPackageInfo]]: """Get release name and package information from version Package info comes from extrinsic metadata (from self.info()) Args: version: Package version (e.g: "0.1.0") Returns: Iterator of tuple (release_name, p_info) """ v = self.info()["versions"][version] assert v["version"] == version url = v["archive_url"] name = v["pubspec"]["name"] filename = f"{name}-{version}.tar.gz" last_modified = v["published"] if "authors" in v["pubspec"]: # TODO: here we have a list of author, see T3887 author = Person.from_fullname(v["pubspec"]["authors"][0].encode()) elif "author" in v["pubspec"] and v["pubspec"]["author"] is not None: author = Person.from_fullname(v["pubspec"]["author"].encode()) else: author = EMPTY_AUTHOR description = v["pubspec"]["description"] p_info = PubDevPackageInfo( name=name, filename=filename, url=url, version=version, last_modified=last_modified, author=author, description=description, ) yield release_name(version), p_info def build_release( self, p_info: PubDevPackageInfo, uncompressed_path: str, directory: Sha1Git ) -> Optional[Release]: # Extract intrinsic metadata from uncompressed_path/pubspec.yaml intrinsic_metadata = extract_intrinsic_metadata(Path(uncompressed_path)) name: str = intrinsic_metadata["name"] version: str = intrinsic_metadata["version"] assert version == p_info.version # author from intrinsic_metadata should not take precedence over the one # returned by the api, see https://dart.dev/tools/pub/pubspec#authorauthors author: Person = p_info.author if "description" in intrinsic_metadata and intrinsic_metadata["description"]: description = intrinsic_metadata["description"] else: description = p_info.description message = ( f"Synthetic release for pub.dev source package {name} " f"version {version}\n\n" f"{description}\n" ) return Release( name=version.encode(), author=author, date=TimestampWithTimezone.from_iso8601(p_info.last_modified), message=message.encode(), target_type=ObjectType.DIRECTORY, target=directory, synthetic=True, ) diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_Autolinker b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_Autolinker index addaa5a..b60f1d8 100644 --- a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_Autolinker +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_Autolinker @@ -1,2 +1,29 @@ -{"name":"Autolinker","latest":{"version":"0.1.1","pubspec":{"version":"0.1.1","homepage":"https://github.com/hackcave","description":"Port of Autolinker.js to dart","name":"Autolinker","author":"hackcave "},"archive_url":"https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz","published":"2014-12-24T22:34:02.534090Z"},"versions":[{"version":"0.1.1","pubspec":{"version":"0.1.1","homepage":"https://github.com/hackcave","description":"Port of Autolinker.js to dart","name":"Autolinker","author":"hackcave "},"archive_url":"https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz","published":"2014-12-24T22:34:02.534090Z"}]} - +{ + "name": "Autolinker", + "latest": { + "version": "0.1.1", + "pubspec": { + "version": "0.1.1", + "homepage": "https://github.com/hackcave", + "description": "Port of Autolinker.js to dart", + "name": "Autolinker", + "author": "hackcave " + }, + "archive_url": "https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz", + "published": "2014-12-24T22:34:02.534090Z" + }, + "versions": [ + { + "version": "0.1.1", + "pubspec": { + "version": "0.1.1", + "homepage": "https://github.com/hackcave", + "description": "Port of Autolinker.js to dart", + "name": "Autolinker", + "author": "hackcave " + }, + "archive_url": "https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz", + "published": "2014-12-24T22:34:02.534090Z" + } + ] +} \ No newline at end of file diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io index e35e397..1d00f61 100644 --- a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io @@ -1,2 +1,93 @@ -{"name":"abstract_io","latest":{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"},"versions":[{"version":"0.1.2+4","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+4","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B4.tar.gz","archive_sha256":"df687ff2a92774db04a28167ccddbfe9c2fc1ea63c6ae05c3236552fe350bb68","published":"2020-08-03T20:14:38.116237Z"},{"version":"0.1.2+5","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+5","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B5.tar.gz","archive_sha256":"fc9199c2f9879d3c0d140c05a2f8c537561af256d98d209b4ee102e8107ec2b9","published":"2020-08-03T21:09:20.329418Z"},{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"}]} - +{ + "name": "abstract_io", + "latest": { + "version": "0.1.2+6", + "pubspec": { + "name": "abstract_io", + "description": "Abstract IO is designed to simplify and generalize saving data both localy and externaly", + "version": "0.1.2+6", + "author": "Anders Groeschel", + "repository": "https://github.com/AndersGroeschel/abstract_io", + "homepage": "https://github.com/AndersGroeschel/abstract_io", + "environment": { + "sdk": ">=2.7.0 <3.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz", + "archive_sha256": "9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18", + "published": "2020-08-03T21:31:05.764846Z" + }, + "versions": [ + { + "version": "0.1.2+4", + "pubspec": { + "name": "abstract_io", + "description": "Abstract IO is designed to simplify and generalize saving data both localy and externaly", + "version": "0.1.2+4", + "author": "Anders Groeschel", + "repository": "https://github.com/AndersGroeschel/abstract_io", + "homepage": "https://github.com/AndersGroeschel/abstract_io", + "environment": { + "sdk": ">=2.7.0 <3.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B4.tar.gz", + "archive_sha256": "df687ff2a92774db04a28167ccddbfe9c2fc1ea63c6ae05c3236552fe350bb68", + "published": "2020-08-03T20:14:38.116237Z" + }, + { + "version": "0.1.2+5", + "pubspec": { + "name": "abstract_io", + "description": "Abstract IO is designed to simplify and generalize saving data both localy and externaly", + "version": "0.1.2+5", + "author": "Anders Groeschel", + "repository": "https://github.com/AndersGroeschel/abstract_io", + "homepage": "https://github.com/AndersGroeschel/abstract_io", + "environment": { + "sdk": ">=2.7.0 <3.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B5.tar.gz", + "archive_sha256": "fc9199c2f9879d3c0d140c05a2f8c537561af256d98d209b4ee102e8107ec2b9", + "published": "2020-08-03T21:09:20.329418Z" + }, + { + "version": "0.1.2+6", + "pubspec": { + "name": "abstract_io", + "description": "Abstract IO is designed to simplify and generalize saving data both localy and externaly", + "version": "0.1.2+6", + "author": "Anders Groeschel", + "repository": "https://github.com/AndersGroeschel/abstract_io", + "homepage": "https://github.com/AndersGroeschel/abstract_io", + "environment": { + "sdk": ">=2.7.0 <3.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz", + "archive_sha256": "9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18", + "published": "2020-08-03T21:31:05.764846Z" + } + ] +} \ No newline at end of file diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_authentication b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_authentication index 5531232..b4b312a 100644 --- a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_authentication +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_authentication @@ -1,2 +1,77 @@ -{"name":"authentication","latest":{"version":"0.0.1","pubspec":{"name":"authentication","description":"Persistent user authentication for Flutter with optional backend API integration.","version":"0.0.1","author":null,"homepage":null,"environment":{"sdk":">=2.7.0 <3.0.0","flutter":">=1.17.0 <2.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}},"dev_dependencies":{"flutter_test":{"sdk":"flutter"}},"flutter":{"plugin":{"platforms":{"some_platform":{"pluginClass":"somePluginClass"}}}}},"archive_url":"https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz","archive_sha256":"0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99","published":"2020-08-13T04:53:34.134687Z"},"versions":[{"version":"0.0.1","pubspec":{"name":"authentication","description":"Persistent user authentication for Flutter with optional backend API integration.","version":"0.0.1","author":null,"homepage":null,"environment":{"sdk":">=2.7.0 <3.0.0","flutter":">=1.17.0 <2.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}},"dev_dependencies":{"flutter_test":{"sdk":"flutter"}},"flutter":{"plugin":{"platforms":{"some_platform":{"pluginClass":"somePluginClass"}}}}},"archive_url":"https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz","archive_sha256":"0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99","published":"2020-08-13T04:53:34.134687Z"}]} - +{ + "name": "authentication", + "latest": { + "version": "0.0.1", + "pubspec": { + "name": "authentication", + "description": "Persistent user authentication for Flutter with optional backend API integration.", + "version": "0.0.1", + "author": null, + "homepage": null, + "environment": { + "sdk": ">=2.7.0 <3.0.0", + "flutter": ">=1.17.0 <2.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + }, + "dev_dependencies": { + "flutter_test": { + "sdk": "flutter" + } + }, + "flutter": { + "plugin": { + "platforms": { + "some_platform": { + "pluginClass": "somePluginClass" + } + } + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz", + "archive_sha256": "0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99", + "published": "2020-08-13T04:53:34.134687Z" + }, + "versions": [ + { + "version": "0.0.1", + "pubspec": { + "name": "authentication", + "description": "Persistent user authentication for Flutter with optional backend API integration.", + "version": "0.0.1", + "author": null, + "homepage": null, + "environment": { + "sdk": ">=2.7.0 <3.0.0", + "flutter": ">=1.17.0 <2.0.0" + }, + "dependencies": { + "flutter": { + "sdk": "flutter" + } + }, + "dev_dependencies": { + "flutter_test": { + "sdk": "flutter" + } + }, + "flutter": { + "plugin": { + "platforms": { + "some_platform": { + "pluginClass": "somePluginClass" + } + } + } + } + }, + "archive_url": "https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz", + "archive_sha256": "0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99", + "published": "2020-08-13T04:53:34.134687Z" + } + ] +} \ No newline at end of file diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_bezier b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_bezier index 70b6d13..efd84a4 100644 --- a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_bezier +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_bezier @@ -1,4 +1,55 @@ -{"name":"bezier","latest":{"version":"1.1.5","pubspec":{"name":"bezier","version":"1.1.5","authors":["Aaron Barrett ","Isaac Barrett "],"description":"A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax . -Live examples can be found at .","homepage":"https://github.com/aab29/bezier.dart","environment":{"sdk":">=2.0.0 <3.0.0"},"dependencies":{"vector_math":"^2.0.0"},"dev_dependencies":{"test":"^1.0.0"}},"archive_url":"https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz","archive_sha256":"cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8","published":"2019-12-22T03:17:30.805225Z"},"versions":[{"version":"1.1.5","pubspec":{"name":"bezier","version":"1.1.5","authors":["Aaron Barrett ","Isaac Barrett "],"description":"A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax . -Live examples can be found at .","homepage":"https://github.com/aab29/bezier.dart","environment":{"sdk":">=2.0.0 <3.0.0"},"dependencies":{"vector_math":"^2.0.0"},"dev_dependencies":{"test":"^1.0.0"}},"archive_url":"https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz","archive_sha256":"cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8","published":"2019-12-22T03:17:30.805225Z"}]} - +{ + "name": "bezier", + "latest": { + "version": "1.1.5", + "pubspec": { + "name": "bezier", + "version": "1.1.5", + "authors": [ + "Aaron Barrett ", + "Isaac Barrett " + ], + "description": "A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax .\nLive examples can be found at .", + "homepage": "https://github.com/aab29/bezier.dart", + "environment": { + "sdk": ">=2.0.0 <3.0.0" + }, + "dependencies": { + "vector_math": "^2.0.0" + }, + "dev_dependencies": { + "test": "^1.0.0" + } + }, + "archive_url": "https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz", + "archive_sha256": "cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8", + "published": "2019-12-22T03:17:30.805225Z" + }, + "versions": [ + { + "version": "1.1.5", + "pubspec": { + "name": "bezier", + "version": "1.1.5", + "authors": [ + "Aaron Barrett ", + "Isaac Barrett " + ], + "description": "A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax .\nLive examples can be found at .", + "homepage": "https://github.com/aab29/bezier.dart", + "environment": { + "sdk": ">=2.0.0 <3.0.0" + }, + "dependencies": { + "vector_math": "^2.0.0" + }, + "dev_dependencies": { + "test": "^1.0.0" + } + }, + "archive_url": "https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz", + "archive_sha256": "cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8", + "published": "2019-12-22T03:17:30.805225Z" + } + ] +} \ No newline at end of file diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_pdf b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_pdf index d636372..c015890 100644 --- a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_pdf +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_pdf @@ -1,2 +1,88 @@ -{"name":"pdf","latest":{"version":"3.8.2","pubspec":{"name":"pdf","description":"A pdf producer for Dart. It can create pdf files for both web or flutter.","homepage":"https://github.com/DavBfr/dart_pdf/tree/master/pdf","repository":"https://github.com/DavBfr/dart_pdf","issue_tracker":"https://github.com/DavBfr/dart_pdf/issues","version":"3.8.2","environment":{"sdk":">=2.12.0 <3.0.0"},"dependencies":{"archive":"^3.1.0","barcode":">=2.2.0 <3.0.0","crypto":"^3.0.0","image":">=3.0.1 <4.0.0","meta":">=1.3.0 <2.0.0","path_parsing":">=0.2.0 <2.0.0","vector_math":"^2.1.0","xml":">=5.1.0 <7.0.0"},"dev_dependencies":{"flutter_lints":"^1.0.4","test":">=1.16.0 <2.0.0"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz","published":"2022-07-25T11:38:25.983876Z"},"versions":[{"version":"1.0.0","pubspec":{"version":"1.0.0","name":"pdf","dependencies":{"ttf_parser":"^1.0.0","vector_math":"^2.0.7","meta":"^1.1.5"},"author":"David PHAM-VAN ","description":"A pdf producer for Dart","homepage":"https://github.com/davbfr/dart_pdf","environment":{"sdk":">=1.8.0 <2.0.0"},"dev_dependencies":{"test":"any"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/1.0.0.tar.gz","published":"2018-07-16T21:12:28.894137Z"},{"version":"3.8.2","pubspec":{"name":"pdf","description":"A pdf producer for Dart. It can create pdf files for both web or flutter.","homepage":"https://github.com/DavBfr/dart_pdf/tree/master/pdf","repository":"https://github.com/DavBfr/dart_pdf","issue_tracker":"https://github.com/DavBfr/dart_pdf/issues","version":"3.8.2","environment":{"sdk":">=2.12.0 <3.0.0"},"dependencies":{"archive":"^3.1.0","barcode":">=2.2.0 <3.0.0","crypto":"^3.0.0","image":">=3.0.1 <4.0.0","meta":">=1.3.0 <2.0.0","path_parsing":">=0.2.0 <2.0.0","vector_math":"^2.1.0","xml":">=5.1.0 <7.0.0"},"dev_dependencies":{"flutter_lints":"^1.0.4","test":">=1.16.0 <2.0.0"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz","published":"2022-07-25T11:38:25.983876Z"}]} - +{ + "name": "pdf", + "latest": { + "version": "3.8.2", + "pubspec": { + "name": "pdf", + "description": "A pdf producer for Dart. It can create pdf files for both web or flutter.", + "homepage": "https://github.com/DavBfr/dart_pdf/tree/master/pdf", + "repository": "https://github.com/DavBfr/dart_pdf", + "issue_tracker": "https://github.com/DavBfr/dart_pdf/issues", + "version": "3.8.2", + "environment": { + "sdk": ">=2.12.0 <3.0.0" + }, + "dependencies": { + "archive": "^3.1.0", + "barcode": ">=2.2.0 <3.0.0", + "crypto": "^3.0.0", + "image": ">=3.0.1 <4.0.0", + "meta": ">=1.3.0 <2.0.0", + "path_parsing": ">=0.2.0 <2.0.0", + "vector_math": "^2.1.0", + "xml": ">=5.1.0 <7.0.0" + }, + "dev_dependencies": { + "flutter_lints": "^1.0.4", + "test": ">=1.16.0 <2.0.0" + } + }, + "archive_url": "https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz", + "published": "2022-07-25T11:38:25.983876Z" + }, + "versions": [ + { + "version": "1.0.0", + "pubspec": { + "version": "1.0.0", + "name": "pdf", + "dependencies": { + "ttf_parser": "^1.0.0", + "vector_math": "^2.0.7", + "meta": "^1.1.5" + }, + "author": "David PHAM-VAN ", + "description": "A pdf producer for Dart", + "homepage": "https://github.com/davbfr/dart_pdf", + "environment": { + "sdk": ">=1.8.0 <2.0.0" + }, + "dev_dependencies": { + "test": "any" + } + }, + "archive_url": "https://pub.dartlang.org/packages/pdf/versions/1.0.0.tar.gz", + "published": "2018-07-16T21:12:28.894137Z" + }, + { + "version": "3.8.2", + "pubspec": { + "name": "pdf", + "description": "A pdf producer for Dart. It can create pdf files for both web or flutter.", + "homepage": "https://github.com/DavBfr/dart_pdf/tree/master/pdf", + "repository": "https://github.com/DavBfr/dart_pdf", + "issue_tracker": "https://github.com/DavBfr/dart_pdf/issues", + "version": "3.8.2", + "environment": { + "sdk": ">=2.12.0 <3.0.0" + }, + "dependencies": { + "archive": "^3.1.0", + "barcode": ">=2.2.0 <3.0.0", + "crypto": "^3.0.0", + "image": ">=3.0.1 <4.0.0", + "meta": ">=1.3.0 <2.0.0", + "path_parsing": ">=0.2.0 <2.0.0", + "vector_math": "^2.1.0", + "xml": ">=5.1.0 <7.0.0" + }, + "dev_dependencies": { + "flutter_lints": "^1.0.4", + "test": ">=1.16.0 <2.0.0" + } + }, + "archive_url": "https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz", + "published": "2022-07-25T11:38:25.983876Z" + } + ] +} \ No newline at end of file diff --git a/swh/loader/package/pubdev/tests/test_pubdev.py b/swh/loader/package/pubdev/tests/test_pubdev.py index a44ff68..0979dfd 100644 --- a/swh/loader/package/pubdev/tests/test_pubdev.py +++ b/swh/loader/package/pubdev/tests/test_pubdev.py @@ -1,270 +1,272 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information + +import pytest + from swh.loader.package.pubdev.loader import PubDevLoader from swh.loader.package.utils import EMPTY_AUTHOR from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes from swh.model.model import ( ObjectType, Person, Release, Snapshot, SnapshotBranch, TargetType, TimestampWithTimezone, ) EXPECTED_PACKAGES = [ { - "url": "https://pub.dev/api/packages/Autolinker", # one version + "url": "https://pub.dev/packages/Autolinker", # one version }, { - "url": "https://pub.dev/api/packages/pdf", # multiple versions + "url": "https://pub.dev/packages/pdf", # multiple versions }, { - "url": "https://pub.dev/api/packages/bezier", # multiple authors + "url": "https://pub.dev/packages/bezier", # multiple authors }, { - "url": "https://pub.dev/api/packages/authentication", # empty author + "url": "https://pub.dev/packages/authentication", # empty author }, { - "url": "https://pub.dev/api/packages/abstract_io", # loose versions names + "url": "https://pub.dev/packages/abstract_io", # loose versions names }, ] def test_get_versions(requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) assert loader.get_versions() == [ "1.0.0", "3.8.2", ] def test_get_loose_versions(requests_mock_datadir, swh_storage): """Sometimes version name does not follow semver""" loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[4]["url"], ) assert loader.get_versions() == ["0.1.2+4", "0.1.2+5", "0.1.2+6"] def test_get_default_version(requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) assert loader.get_default_version() == "3.8.2" def test_pubdev_loader_load_one_version(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[0]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "245092931ba809e6c54ebda8f865fb5a969a4134" expected_release_id = "919f267ea050539606344d49d14bf594c4386e5a" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/0.1.1": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/0.1.1", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert { "content": 1, "directory": 1, "origin": 1, "origin_visit": 1, "release": 1, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( name=b"0.1.1", message=b"Synthetic release for pub.dev source package Autolinker version" b" 0.1.1\n\nPort of Autolinker.js to dart\n", target=hash_to_bytes("3fb6d4f2c0334d1604357ae92b2dd38a55a78194"), target_type=ObjectType.DIRECTORY, synthetic=True, author=Person( fullname=b"hackcave ", name=b"hackcave", email=b"hackers@hackcave.org", ), date=TimestampWithTimezone.from_iso8601("2014-12-24T22:34:02.534090+00:00"), id=hash_to_bytes(expected_release_id), ) assert_last_visit_matches( swh_storage, url=EXPECTED_PACKAGES[0]["url"], status="full", type="pubdev", snapshot=expected_snapshot.id, ) def test_pubdev_loader_load_multiple_versions( datadir, requests_mock_datadir, swh_storage ): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "43d5b68a9fa973aa95e56916aaef70841ccbc2a0" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/1.0.0": SnapshotBranch( target=hash_to_bytes("fbf8e40af675096681954553d737861e10b57216"), target_type=TargetType.RELEASE, ), b"releases/3.8.2": SnapshotBranch( target=hash_to_bytes("627a5d586e3fb4e7319b17f1aee268fe2fb8e01c"), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/3.8.2", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert { "content": 1 + 1, "directory": 1 + 1, "origin": 1, "origin_visit": 1, "release": 1 + 1, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats assert_last_visit_matches( swh_storage, url=EXPECTED_PACKAGES[1]["url"], status="full", type="pubdev", snapshot=expected_snapshot.id, ) def test_pubdev_loader_multiple_authors(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[2]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "4fa9f19d1d6ccc70921c8c50b278f510db63aa36" expected_release_id = "538c98fd69a42d8d0561a7ca95b354de2143a3ab" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/1.1.5": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/1.1.5", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) release = swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] assert release.author == Person( fullname=b"Aaron Barrett ", name=b"Aaron Barrett", email=b"aaron@aaronbarrett.com", ) def test_pubdev_loader_empty_author(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[3]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "0c7fa6b9fced23c648d2093ad5597622683f8aed" expected_release_id = "7d8c05181069aa1049a3f0bc1d13bedc34625d47" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/0.0.1": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/0.0.1", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) release = swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] assert release.author == EMPTY_AUTHOR -def test_pubdev_invalid_origin(swh_storage, requests_mock_datadir): - loader = PubDevLoader( - swh_storage, - "http://nowhere/api/packages/42", - ) +def test_pubdev_invalid_origin(swh_storage): - load_status = loader.load() - assert load_status["status"] == "failed" + with pytest.raises(AssertionError): + PubDevLoader( + swh_storage, + "http://nowhere/api/packages/42", + ) diff --git a/swh/loader/package/pubdev/tests/test_tasks.py b/swh/loader/package/pubdev/tests/test_tasks.py index 6c7bb57..c5b2ce7 100644 --- a/swh/loader/package/pubdev/tests/test_tasks.py +++ b/swh/loader/package/pubdev/tests/test_tasks.py @@ -1,23 +1,23 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information def test_tasks_pubdev_loader( mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config ): mock_load = mocker.patch("swh.loader.package.pubdev.loader.PubDevLoader.load") mock_load.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.package.pubdev.tasks.LoadPubDev", kwargs=dict( - url="some-url/api/packages/some-package", + url="https://pub.dev/packages/some-package", ), ) assert res res.wait() assert res.successful() assert mock_load.called assert res.result == {"status": "eventful"}