diff --git a/swh/loader/package/pubdev/loader.py b/swh/loader/package/pubdev/loader.py index f5e161c..e278b0d 100644 --- a/swh/loader/package/pubdev/loader.py +++ b/swh/loader/package/pubdev/loader.py @@ -1,188 +1,188 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from distutils.version import StrictVersion +from distutils.version import LooseVersion import json from pathlib import Path from typing import Any, Dict, Iterator, Optional, Sequence, Tuple import attr import yaml from swh.loader.package.loader import BasePackageInfo, PackageLoader from swh.loader.package.utils import ( EMPTY_AUTHOR, Person, api_info, cached_method, release_name, ) from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone from swh.storage.interface import StorageInterface @attr.s class PubDevPackageInfo(BasePackageInfo): name = attr.ib(type=str) """Name of the package""" version = attr.ib(type=str) """Current version""" last_modified = attr.ib(type=str) """Last modified date as release date""" author = attr.ib(type=Person) """Author""" description = attr.ib(type=str) """Description""" def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: """Extract intrinsic metadata from pubspec.yaml file at dir_path. Each pub.dev package version has a pubspec.yaml file at the root of the archive. See https://dart.dev/tools/pub/pubspec for pubspec specifications. Args: dir_path: A directory on disk where a pubspec.yaml must be present Returns: A dict mapping from yaml parser """ pubspec_path = dir_path / "pubspec.yaml" return yaml.safe_load(pubspec_path.read_text()) class PubDevLoader(PackageLoader[PubDevPackageInfo]): visit_type = "pubdev" def __init__( self, storage: StorageInterface, url: str, **kwargs, ): super().__init__(storage=storage, url=url, **kwargs) self.url = url def _raw_info(self) -> bytes: return api_info(self.url) @cached_method def info(self) -> Dict: """Return the project metadata information (fetched from pub.dev registry)""" # Use strict=False in order to correctly manage case where \n is present in a string info = json.loads(self._raw_info(), strict=False) # Arrange versions list as a new dict with `version` as key versions = {v["version"]: v for v in info["versions"]} info["versions"] = versions return info def get_versions(self) -> Sequence[str]: """Get all released versions of a PubDev package Returns: A sequence of versions Example:: ["0.1.1", "0.10.2"] """ versions = list(self.info()["versions"].keys()) - versions.sort(key=StrictVersion) + versions.sort(key=LooseVersion) return versions def get_default_version(self) -> str: """Get the newest release version of a PubDev package Returns: A string representing a version Example:: "0.1.2" """ latest = self.info()["latest"] return latest["version"] def get_package_info(self, version: str) -> Iterator[Tuple[str, PubDevPackageInfo]]: """Get release name and package information from version Package info comes from extrinsic metadata (from self.info()) Args: version: Package version (e.g: "0.1.0") Returns: Iterator of tuple (release_name, p_info) """ v = self.info()["versions"][version] assert v["version"] == version url = v["archive_url"] name = v["pubspec"]["name"] filename = f"{name}-{version}.tar.gz" last_modified = v["published"] if "authors" in v["pubspec"]: # TODO: here we have a list of author, see T3887 author = Person.from_fullname(v["pubspec"]["authors"][0].encode()) elif "author" in v["pubspec"] and v["pubspec"]["author"] is not None: author = Person.from_fullname(v["pubspec"]["author"].encode()) else: author = EMPTY_AUTHOR description = v["pubspec"]["description"] p_info = PubDevPackageInfo( name=name, filename=filename, url=url, version=version, last_modified=last_modified, author=author, description=description, ) yield release_name(version), p_info def build_release( self, p_info: PubDevPackageInfo, uncompressed_path: str, directory: Sha1Git ) -> Optional[Release]: # Extract intrinsic metadata from uncompressed_path/pubspec.yaml intrinsic_metadata = extract_intrinsic_metadata(Path(uncompressed_path)) name: str = intrinsic_metadata["name"] version: str = intrinsic_metadata["version"] assert version == p_info.version # author from intrinsic_metadata should not take precedence over the one # returned by the api, see https://dart.dev/tools/pub/pubspec#authorauthors author: Person = p_info.author if "description" in intrinsic_metadata and intrinsic_metadata["description"]: description = intrinsic_metadata["description"] else: description = p_info.description message = ( f"Synthetic release for pub.dev source package {name} " f"version {version}\n\n" f"{description}\n" ) return Release( name=version.encode(), author=author, date=TimestampWithTimezone.from_iso8601(p_info.last_modified), message=message.encode(), target_type=ObjectType.DIRECTORY, target=directory, synthetic=True, ) diff --git a/swh/loader/package/pubdev/tests/data/fake_pubdev.sh b/swh/loader/package/pubdev/tests/data/fake_pubdev.sh index e65b394..c4c33dd 100644 --- a/swh/loader/package/pubdev/tests/data/fake_pubdev.sh +++ b/swh/loader/package/pubdev/tests/data/fake_pubdev.sh @@ -1,191 +1,194 @@ #!/usr/bin/env bash # Script to generate fake pub.dev http api response and fake Dart or FLutter packages archives as .tar.gz. set -euo pipefail # Create directories readonly TMP=tmp_dir/pubdev readonly BASE_API=https_pub.dev readonly BASE_ARCHIVES=https_pub.dartlang.org mkdir -p $TMP mkdir -p $BASE_API mkdir -p $BASE_ARCHIVES # http api response as json echo -e '''{"name":"authentication","latest":{"version":"0.0.1","pubspec":{"name":"authentication","description":"Persistent user authentication for Flutter with optional backend API integration.","version":"0.0.1","author":null,"homepage":null,"environment":{"sdk":">=2.7.0 <3.0.0","flutter":">=1.17.0 <2.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}},"dev_dependencies":{"flutter_test":{"sdk":"flutter"}},"flutter":{"plugin":{"platforms":{"some_platform":{"pluginClass":"somePluginClass"}}}}},"archive_url":"https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz","archive_sha256":"0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99","published":"2020-08-13T04:53:34.134687Z"},"versions":[{"version":"0.0.1","pubspec":{"name":"authentication","description":"Persistent user authentication for Flutter with optional backend API integration.","version":"0.0.1","author":null,"homepage":null,"environment":{"sdk":">=2.7.0 <3.0.0","flutter":">=1.17.0 <2.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}},"dev_dependencies":{"flutter_test":{"sdk":"flutter"}},"flutter":{"plugin":{"platforms":{"some_platform":{"pluginClass":"somePluginClass"}}}}},"archive_url":"https://pub.dartlang.org/packages/authentication/versions/0.0.1.tar.gz","archive_sha256":"0179334b346cb67e4e6e3c905e5cc5c8e488a45ebd99fd2be3a7e0476d620d99","published":"2020-08-13T04:53:34.134687Z"}]} ''' > $BASE_API/api_packages_authentication echo -e '''{"name":"Autolinker","latest":{"version":"0.1.1","pubspec":{"version":"0.1.1","homepage":"https://github.com/hackcave","description":"Port of Autolinker.js to dart","name":"Autolinker","author":"hackcave "},"archive_url":"https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz","published":"2014-12-24T22:34:02.534090Z"},"versions":[{"version":"0.1.1","pubspec":{"version":"0.1.1","homepage":"https://github.com/hackcave","description":"Port of Autolinker.js to dart","name":"Autolinker","author":"hackcave "},"archive_url":"https://pub.dartlang.org/packages/Autolinker/versions/0.1.1.tar.gz","published":"2014-12-24T22:34:02.534090Z"}]} ''' > ${BASE_API}/api_packages_Autolinker echo -e '''{"name":"bezier","latest":{"version":"1.1.5","pubspec":{"name":"bezier","version":"1.1.5","authors":["Aaron Barrett ","Isaac Barrett "],"description":"A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax .\nLive examples can be found at .","homepage":"https://github.com/aab29/bezier.dart","environment":{"sdk":">=2.0.0 <3.0.0"},"dependencies":{"vector_math":"^2.0.0"},"dev_dependencies":{"test":"^1.0.0"}},"archive_url":"https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz","archive_sha256":"cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8","published":"2019-12-22T03:17:30.805225Z"},"versions":[{"version":"1.1.5","pubspec":{"name":"bezier","version":"1.1.5","authors":["Aaron Barrett ","Isaac Barrett "],"description":"A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax .\nLive examples can be found at .","homepage":"https://github.com/aab29/bezier.dart","environment":{"sdk":">=2.0.0 <3.0.0"},"dependencies":{"vector_math":"^2.0.0"},"dev_dependencies":{"test":"^1.0.0"}},"archive_url":"https://pub.dartlang.org/packages/bezier/versions/1.1.5.tar.gz","archive_sha256":"cc5da2fa927b5d347550f78d456cd984b7df78a7f0405119cdab12111e2f9ee8","published":"2019-12-22T03:17:30.805225Z"}]} ''' > ${BASE_API}/api_packages_bezier echo -e '''{"name":"pdf","latest":{"version":"3.8.2","pubspec":{"name":"pdf","description":"A pdf producer for Dart. It can create pdf files for both web or flutter.","homepage":"https://github.com/DavBfr/dart_pdf/tree/master/pdf","repository":"https://github.com/DavBfr/dart_pdf","issue_tracker":"https://github.com/DavBfr/dart_pdf/issues","version":"3.8.2","environment":{"sdk":">=2.12.0 <3.0.0"},"dependencies":{"archive":"^3.1.0","barcode":">=2.2.0 <3.0.0","crypto":"^3.0.0","image":">=3.0.1 <4.0.0","meta":">=1.3.0 <2.0.0","path_parsing":">=0.2.0 <2.0.0","vector_math":"^2.1.0","xml":">=5.1.0 <7.0.0"},"dev_dependencies":{"flutter_lints":"^1.0.4","test":">=1.16.0 <2.0.0"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz","published":"2022-07-25T11:38:25.983876Z"},"versions":[{"version":"1.0.0","pubspec":{"version":"1.0.0","name":"pdf","dependencies":{"ttf_parser":"^1.0.0","vector_math":"^2.0.7","meta":"^1.1.5"},"author":"David PHAM-VAN ","description":"A pdf producer for Dart","homepage":"https://github.com/davbfr/dart_pdf","environment":{"sdk":">=1.8.0 <2.0.0"},"dev_dependencies":{"test":"any"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/1.0.0.tar.gz","published":"2018-07-16T21:12:28.894137Z"},{"version":"3.8.2","pubspec":{"name":"pdf","description":"A pdf producer for Dart. It can create pdf files for both web or flutter.","homepage":"https://github.com/DavBfr/dart_pdf/tree/master/pdf","repository":"https://github.com/DavBfr/dart_pdf","issue_tracker":"https://github.com/DavBfr/dart_pdf/issues","version":"3.8.2","environment":{"sdk":">=2.12.0 <3.0.0"},"dependencies":{"archive":"^3.1.0","barcode":">=2.2.0 <3.0.0","crypto":"^3.0.0","image":">=3.0.1 <4.0.0","meta":">=1.3.0 <2.0.0","path_parsing":">=0.2.0 <2.0.0","vector_math":"^2.1.0","xml":">=5.1.0 <7.0.0"},"dev_dependencies":{"flutter_lints":"^1.0.4","test":">=1.16.0 <2.0.0"}},"archive_url":"https://pub.dartlang.org/packages/pdf/versions/3.8.2.tar.gz","published":"2022-07-25T11:38:25.983876Z"}]} ''' > ${BASE_API}/api_packages_pdf +echo -e '''{"name":"abstract_io","latest":{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"},"versions":[{"version":"0.1.2+4","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+4","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B4.tar.gz","archive_sha256":"df687ff2a92774db04a28167ccddbfe9c2fc1ea63c6ae05c3236552fe350bb68","published":"2020-08-03T20:14:38.116237Z"},{"version":"0.1.2+5","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+5","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B5.tar.gz","archive_sha256":"fc9199c2f9879d3c0d140c05a2f8c537561af256d98d209b4ee102e8107ec2b9","published":"2020-08-03T21:09:20.329418Z"},{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"}]} +''' > ${BASE_API}/api_packages_abstract_io + # Dart package a pubspec.yaml file at thier root. Generate some of them. mkdir -p ${TMP}/packages_authentication_versions_0.0.1 echo -e '''name: authentication description: Persistent user authentication for Flutter with optional backend API integration. version: 0.0.1 author: homepage: environment: sdk: ">=2.7.0 <3.0.0" flutter: ">=1.17.0 <2.0.0" dependencies: flutter: sdk: flutter dev_dependencies: flutter_test: sdk: flutter # For information on the generic Dart part of this file, see the # following page: https://dart.dev/tools/pub/pubspec # The following section is specific to Flutter. flutter: # This section identifies this Flutter project as a plugin project. # The 'pluginClass' and Android 'package' identifiers should not ordinarily # be modified. They are used by the tooling to maintain consistency when # adding or updating assets for this project. plugin: platforms: # This plugin project was generated without specifying any # platforms with the `--platform` argument. If you see the `fake_platform` map below, remove it and # then add platforms following the instruction here: # https://flutter.dev/docs/development/packages-and-plugins/developing-packages#plugin-platforms # ------------------- some_platform: pluginClass: somePluginClass # ------------------- # To add assets to your plugin package, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg # - images/a_dot_ham.jpeg # # For details regarding assets in packages, see # https://flutter.dev/assets-and-images/#from-packages # # An image asset can refer to one or more resolution-specific "variants", see # https://flutter.dev/assets-and-images/#resolution-aware. # To add custom fonts to your plugin package, add a fonts section here, # in this "flutter" section. Each entry in this list should have a # "family" key with the font family name, and a "fonts" key with a # list giving the asset and other descriptors for the font. For # example: # fonts: # - family: Schyler # fonts: # - asset: fonts/Schyler-Regular.ttf # - asset: fonts/Schyler-Italic.ttf # style: italic # - family: Trajan Pro # fonts: # - asset: fonts/TrajanPro.ttf # - asset: fonts/TrajanPro_Bold.ttf # weight: 700 # # For details regarding fonts in packages, see # https://flutter.dev/custom-fonts/#from-packages ''' > ${TMP}/packages_authentication_versions_0.0.1/pubspec.yaml mkdir -p ${TMP}/packages_autolinker_versions_0.1.1 echo -e '''name: Autolinker version: 0.1.1 author: hackcave homepage: https://github.com/hackcave description: Port of Autolinker.js to dart ''' > ${TMP}/packages_autolinker_versions_0.1.1/pubspec.yaml mkdir -p ${TMP}/packages_bezier_versions_1.1.5 echo -e '''name: bezier version: 1.1.5 authors: - Aaron Barrett - Isaac Barrett description: >- A 2D Bézier curve math library. Based heavily on the work of @TheRealPomax . Live examples can be found at . homepage: https://github.com/aab29/bezier.dart environment: sdk: ">=2.0.0 <3.0.0" dependencies: vector_math: ^2.0.0 dev_dependencies: test: ^1.0.0 ''' > ${TMP}/packages_bezier_versions_1.1.5/pubspec.yaml mkdir -p ${TMP}/packages_pdf_versions_1.0.0 echo -e '''name: pdf author: David PHAM-VAN description: A pdf producer for Dart homepage: https://github.com/davbfr/dart_pdf version: 1.0.0 environment: sdk: ">=1.8.0 <2.0.0" dependencies: meta: "^1.1.5" ttf_parser: "^1.0.0" vector_math: "^2.0.7" dev_dependencies: test: any ''' > ${TMP}/packages_pdf_versions_1.0.0/pubspec.yaml mkdir -p ${TMP}/packages_pdf_versions_3.8.2 echo -e '''name: pdf description: A pdf producer for Dart. It can create pdf files for both web or flutter. homepage: https://github.com/DavBfr/dart_pdf/tree/master/pdf repository: https://github.com/DavBfr/dart_pdf issue_tracker: https://github.com/DavBfr/dart_pdf/issues version: 3.8.2 environment: sdk: ">=2.12.0 <3.0.0" dependencies: archive: ^3.1.0 barcode: ">=2.2.0 <3.0.0" crypto: ^3.0.0 image: ">=3.0.1 <4.0.0" meta: ">=1.3.0 <2.0.0" path_parsing: ">=0.2.0 <2.0.0" vector_math: ^2.1.0 xml: ">=5.1.0 <7.0.0" dev_dependencies: flutter_lints: ^1.0.4 test: ">=1.16.0 <2.0.0" ''' > ${TMP}/packages_pdf_versions_3.8.2/pubspec.yaml cd $TMP tar -czf packages_authentication_versions_0.0.1.tar.gz -C packages_authentication_versions_0.0.1 . tar -czf packages_Autolinker_versions_0.1.1.tar.gz -C packages_autolinker_versions_0.1.1 . tar -czf packages_bezier_versions_1.1.5.tar.gz -C packages_bezier_versions_1.1.5 . tar -czf packages_pdf_versions_1.0.0.tar.gz -C packages_pdf_versions_1.0.0 . tar -czf packages_pdf_versions_3.8.2.tar.gz -C packages_pdf_versions_3.8.2 . # Move .tar.gz archives to a servable directory mv *.tar.gz ../../$BASE_ARCHIVES # Clean up removing tmp_dir cd ../../ rm -r tmp_dir/ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_Autolinker_versions_0.1.1.tar.gz b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_Autolinker_versions_0.1.1.tar.gz index 5dcef31..5cdf2dd 100644 Binary files a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_Autolinker_versions_0.1.1.tar.gz and b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_Autolinker_versions_0.1.1.tar.gz differ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_authentication_versions_0.0.1.tar.gz b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_authentication_versions_0.0.1.tar.gz index a4ca17e..4338014 100644 Binary files a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_authentication_versions_0.0.1.tar.gz and b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_authentication_versions_0.0.1.tar.gz differ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_bezier_versions_1.1.5.tar.gz b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_bezier_versions_1.1.5.tar.gz index d9d15ae..5a92354 100644 Binary files a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_bezier_versions_1.1.5.tar.gz and b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_bezier_versions_1.1.5.tar.gz differ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_1.0.0.tar.gz b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_1.0.0.tar.gz index c8b5dfb..d30d19f 100644 Binary files a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_1.0.0.tar.gz and b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_1.0.0.tar.gz differ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_3.8.2.tar.gz b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_3.8.2.tar.gz index 5e722c6..45e37e8 100644 Binary files a/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_3.8.2.tar.gz and b/swh/loader/package/pubdev/tests/data/https_pub.dartlang.org/packages_pdf_versions_3.8.2.tar.gz differ diff --git a/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io new file mode 100644 index 0000000..e35e397 --- /dev/null +++ b/swh/loader/package/pubdev/tests/data/https_pub.dev/api_packages_abstract_io @@ -0,0 +1,2 @@ +{"name":"abstract_io","latest":{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"},"versions":[{"version":"0.1.2+4","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+4","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B4.tar.gz","archive_sha256":"df687ff2a92774db04a28167ccddbfe9c2fc1ea63c6ae05c3236552fe350bb68","published":"2020-08-03T20:14:38.116237Z"},{"version":"0.1.2+5","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+5","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B5.tar.gz","archive_sha256":"fc9199c2f9879d3c0d140c05a2f8c537561af256d98d209b4ee102e8107ec2b9","published":"2020-08-03T21:09:20.329418Z"},{"version":"0.1.2+6","pubspec":{"name":"abstract_io","description":"Abstract IO is designed to simplify and generalize saving data both localy and externaly","version":"0.1.2+6","author":"Anders Groeschel","repository":"https://github.com/AndersGroeschel/abstract_io","homepage":"https://github.com/AndersGroeschel/abstract_io","environment":{"sdk":">=2.7.0 <3.0.0"},"dependencies":{"flutter":{"sdk":"flutter"}}},"archive_url":"https://pub.dartlang.org/packages/abstract_io/versions/0.1.2%2B6.tar.gz","archive_sha256":"9557fd384730d92a046cfccdff9625f2d646657219d5a0e447cb7eb0fdf90f18","published":"2020-08-03T21:31:05.764846Z"}]} + diff --git a/swh/loader/package/pubdev/tests/test_pubdev.py b/swh/loader/package/pubdev/tests/test_pubdev.py index 207acd5..a44ff68 100644 --- a/swh/loader/package/pubdev/tests/test_pubdev.py +++ b/swh/loader/package/pubdev/tests/test_pubdev.py @@ -1,258 +1,270 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.loader.package.pubdev.loader import PubDevLoader from swh.loader.package.utils import EMPTY_AUTHOR from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes from swh.model.model import ( ObjectType, Person, Release, Snapshot, SnapshotBranch, TargetType, TimestampWithTimezone, ) EXPECTED_PACKAGES = [ { "url": "https://pub.dev/api/packages/Autolinker", # one version }, { "url": "https://pub.dev/api/packages/pdf", # multiple versions }, { "url": "https://pub.dev/api/packages/bezier", # multiple authors }, { "url": "https://pub.dev/api/packages/authentication", # empty author }, + { + "url": "https://pub.dev/api/packages/abstract_io", # loose versions names + }, ] def test_get_versions(requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) assert loader.get_versions() == [ "1.0.0", "3.8.2", ] +def test_get_loose_versions(requests_mock_datadir, swh_storage): + """Sometimes version name does not follow semver""" + loader = PubDevLoader( + swh_storage, + url=EXPECTED_PACKAGES[4]["url"], + ) + assert loader.get_versions() == ["0.1.2+4", "0.1.2+5", "0.1.2+6"] + + def test_get_default_version(requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) assert loader.get_default_version() == "3.8.2" def test_pubdev_loader_load_one_version(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[0]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "245092931ba809e6c54ebda8f865fb5a969a4134" expected_release_id = "919f267ea050539606344d49d14bf594c4386e5a" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/0.1.1": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/0.1.1", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert { "content": 1, "directory": 1, "origin": 1, "origin_visit": 1, "release": 1, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( name=b"0.1.1", message=b"Synthetic release for pub.dev source package Autolinker version" b" 0.1.1\n\nPort of Autolinker.js to dart\n", target=hash_to_bytes("3fb6d4f2c0334d1604357ae92b2dd38a55a78194"), target_type=ObjectType.DIRECTORY, synthetic=True, author=Person( fullname=b"hackcave ", name=b"hackcave", email=b"hackers@hackcave.org", ), date=TimestampWithTimezone.from_iso8601("2014-12-24T22:34:02.534090+00:00"), id=hash_to_bytes(expected_release_id), ) assert_last_visit_matches( swh_storage, url=EXPECTED_PACKAGES[0]["url"], status="full", type="pubdev", snapshot=expected_snapshot.id, ) def test_pubdev_loader_load_multiple_versions( datadir, requests_mock_datadir, swh_storage ): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[1]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "43d5b68a9fa973aa95e56916aaef70841ccbc2a0" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/1.0.0": SnapshotBranch( target=hash_to_bytes("fbf8e40af675096681954553d737861e10b57216"), target_type=TargetType.RELEASE, ), b"releases/3.8.2": SnapshotBranch( target=hash_to_bytes("627a5d586e3fb4e7319b17f1aee268fe2fb8e01c"), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/3.8.2", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) stats = get_stats(swh_storage) assert { "content": 1 + 1, "directory": 1 + 1, "origin": 1, "origin_visit": 1, "release": 1 + 1, "revision": 0, "skipped_content": 0, "snapshot": 1, } == stats assert_last_visit_matches( swh_storage, url=EXPECTED_PACKAGES[1]["url"], status="full", type="pubdev", snapshot=expected_snapshot.id, ) def test_pubdev_loader_multiple_authors(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[2]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "4fa9f19d1d6ccc70921c8c50b278f510db63aa36" expected_release_id = "538c98fd69a42d8d0561a7ca95b354de2143a3ab" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/1.1.5": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/1.1.5", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) release = swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] assert release.author == Person( fullname=b"Aaron Barrett ", name=b"Aaron Barrett", email=b"aaron@aaronbarrett.com", ) def test_pubdev_loader_empty_author(datadir, requests_mock_datadir, swh_storage): loader = PubDevLoader( swh_storage, url=EXPECTED_PACKAGES[3]["url"], ) load_status = loader.load() assert load_status["status"] == "eventful" assert load_status["snapshot_id"] is not None expected_snapshot_id = "0c7fa6b9fced23c648d2093ad5597622683f8aed" expected_release_id = "7d8c05181069aa1049a3f0bc1d13bedc34625d47" assert expected_snapshot_id == load_status["snapshot_id"] expected_snapshot = Snapshot( id=hash_to_bytes(load_status["snapshot_id"]), branches={ b"releases/0.0.1": SnapshotBranch( target=hash_to_bytes(expected_release_id), target_type=TargetType.RELEASE, ), b"HEAD": SnapshotBranch( target=b"releases/0.0.1", target_type=TargetType.ALIAS, ), }, ) check_snapshot(expected_snapshot, swh_storage) release = swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] assert release.author == EMPTY_AUTHOR def test_pubdev_invalid_origin(swh_storage, requests_mock_datadir): loader = PubDevLoader( swh_storage, "http://nowhere/api/packages/42", ) load_status = loader.load() assert load_status["status"] == "failed"