diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d0fb19c..839c429 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,42 +1,43 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.1.0 hooks: - id: trailing-whitespace - id: check-json - id: check-yaml - repo: https://gitlab.com/pycqa/flake8 rev: 4.0.1 hooks: - id: flake8 additional_dependencies: [flake8-bugbear==22.3.23] - repo: https://github.com/codespell-project/codespell rev: v2.1.0 hooks: - id: codespell name: Check source code spelling exclude: ^(swh/loader/package/.*[/]+tests/data/.*)$ + args: [-L crate] entry: codespell --ignore-words-list=iff stages: [commit] - repo: local hooks: - id: mypy name: mypy entry: mypy args: [swh] pass_filenames: false language: system types: [python] - repo: https://github.com/PyCQA/isort rev: 5.10.1 hooks: - id: isort - repo: https://github.com/python/black rev: 22.3.0 hooks: - id: black diff --git a/docs/package-loader-specifications.rst b/docs/package-loader-specifications.rst index b53577b..caad4ac 100644 --- a/docs/package-loader-specifications.rst +++ b/docs/package-loader-specifications.rst @@ -1,124 +1,133 @@ .. _package-loader-specifications: Package loader specifications ============================= Release fields -------------- Here is an overview of the fields (+ internal version name + branch name) used by each package loader, after D6616: .. list-table:: Fields used by each package loader :header-rows: 1 * - Loader - internal version - branch name - name - message - synthetic - author - date - Notes * - archive - passed as arg - ``release_name(​version)`` - =version - "Synthetic release for archive at {p_info.url}\n" - true - "" - passed as arg - * - cran - ``metadata.get(​"Version", passed as arg)`` - ``release_name(​version)`` - =version - standard message - true - ``metadata.get(​"Maintainer", "")`` - ``metadata.get(​"Date")`` - metadata is intrinsic + * - crates + - passed as arg + - ``release_name(​version, filename)`` + - =version + - Synthetic release for Crate source package {p_info.name} version {p_info.version} {description} + - true + - from intrinsic metadata + - from extrinsic metadata + - ``i_metadata`` for intrinsic metadata, ``e_metadata`` for extrinsic metadata * - debian - =``version`` - ``release_name(​version)`` - =``i_version`` - standard message (using ``i_version``) - true - ``metadata​.changelog​.person`` - ``metadata​.changelog​.date`` - metadata is intrinsic. Old revisions have ``dsc`` as type ``i_version`` is the intrinsic version (eg. ``0.7.2-3``) while ``version`` contains the debian suite name (eg. ``stretch/contrib/0.7.2-3``) and is passed as arg * - deposit - HEAD - only HEAD - HEAD - "{client}: Deposit {id} in collection {collection}\n" - true - original author - ```` from SWORD XML - revisions had parents * - maven-loader - passed as arg - HEAD - ``release_name(version)`` - "Synthetic release for archive at {p_info.url}\n" - true - "" - passed as arg - Only one artefact per url (jar/zip src) * - nixguix - URL - URL - URL - None - true - "" - None - it's the URL of the artifact referenced by the derivation * - npm - ``metadata​["version"]`` - ``release_name(​version)`` - =version - standard message - true - from int metadata or "" - from ext metadata or None - * - opam - as given by opam - "{opam_package}​.{version}" - =version - standard message - true - from metadata - None - "{self.opam_package}​.{version}" matches the version names used by opam's backend. metadata is extrinsic * - pypi - ``metadata​["version"]`` - ``release_name(​version)`` or ``release_name(​version, filename)`` - =version - ``metadata[​'comment_text']}`` or standard message - true - from int metadata or "" - from ext metadata or None - metadata is intrinsic using this function:: def release_name(version: str, filename: Optional[str] = None) -> str: if filename: return "releases/%s/%s" % (version, filename) return "releases/%s" % version and "standard message" being:: msg = ( f"Synthetic release for {PACKAGE_MANAGER} source package {name} " f"version {version}\n" ) The ``target_type`` field is always ``dir``, and the target the id of a directory loaded by unpacking a tarball/zip file/... diff --git a/requirements.txt b/requirements.txt index 3be0d4a..644a170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,12 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html psutil requests iso8601 pkginfo python-debian python-dateutil typing-extensions +toml diff --git a/setup.py b/setup.py index 81f0481..a4f4d95 100755 --- a/setup.py +++ b/setup.py @@ -1,81 +1,82 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.rst"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.loader.core", description="Software Heritage Base Loader", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DLDBASE", packages=find_packages(), # packages's modules scripts=[], # scripts to package install_requires=parse_requirements() + parse_requirements("swh"), setup_requires=["setuptools-scm"], use_scm_version=True, extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" [swh.cli.subcommands] loader=swh.loader.cli [swh.workers] loader.archive=swh.loader.package.archive:register loader.cran=swh.loader.package.cran:register + loader.crates=swh.loader.package.crates:register loader.debian=swh.loader.package.debian:register loader.deposit=swh.loader.package.deposit:register loader.nixguix=swh.loader.package.nixguix:register loader.npm=swh.loader.package.npm:register loader.opam=swh.loader.package.opam:register loader.pypi=swh.loader.package.pypi:register loader.maven=swh.loader.package.maven:register """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-loader-core", "Documentation": "https://docs.softwareheritage.org/devel/swh-loader-core/", }, ) diff --git a/swh/loader/package/crates/__init__.py b/swh/loader/package/crates/__init__.py new file mode 100644 index 0000000..ec7214c --- /dev/null +++ b/swh/loader/package/crates/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from .loader import CratesLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": CratesLoader, + } diff --git a/swh/loader/package/crates/loader.py b/swh/loader/package/crates/loader.py new file mode 100644 index 0000000..4b59015 --- /dev/null +++ b/swh/loader/package/crates/loader.py @@ -0,0 +1,355 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from distutils.version import StrictVersion +import json +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple +from urllib.parse import urlparse + +import attr +import toml +from typing_extensions import TypedDict + +from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.loader.package.utils import api_info, cached_method, release_name +from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone +from swh.storage.interface import StorageInterface + + +class ExtrinsicPackageMetadata(TypedDict): + """Data structure for package extrinsic metadata pulled from http api endpoint. + + We set only the keys we need according to what is available when querying + https://crates.io/api/v1/crates/, where `name` is the name of the crate + package. + + Json response example for https://crates.io/api/v1/crates/hg-core : + + .. literalinclude:: ../../swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core + + Usage example: + + .. code-block:: python + + e_metadata = ExtrinsicPackageMetadata(**self.info()) + + """ # noqa + + categories: List[Dict[Any, Any]] + """Related categories""" + + crate: Dict[Any, Any] + """Crate project information""" + + keywords: List[Any] + """Keywords""" + + versions: List[Dict[Any, Any]] + """A list of released versions for a crate""" + + +class ExtrinsicVersionPackageMetadata(TypedDict): + """Data structure for specific package version extrinsic metadata, pulled + from http api endpoint. + + Similar to `ExtrinsicPackageMetadata` in its usage, but we flatten the data + related to a specific version. + """ + + crate: str + """The package name""" + + crate_size: int + """The package size""" + + created_at: str + """First released at""" + + downloads: str + """Number of downloads""" + + license: str + """Package license""" + + num: str + """Package version""" + + published_by: Dict[Any, Any] + """Publishers information""" + + updated_at: str + """Last update""" + + yanked: bool + """Is that version yanked? (yanked means release-level deprecation)""" + + +class IntrinsicPackageMetadata(TypedDict): + """Data structure for specific package version intrinsic metadata. + + Data is extracted from the crate package's .toml file. Then the data of the + 'package' entry is flattened. + + Cargo.toml file content example: + + .. code-block:: toml + + [package] + name = "hg-core" + version = "0.0.1" + authors = ["Georges Racinet "] + description = "Mercurial pure Rust core library, with no assumption on + Python bindings (FFI)" + homepage = "https://mercurial-scm.org" + license = "GPL-2.0-or-later" + repository = "https://www.mercurial-scm.org/repo/hg" + + [lib] + name = "hg" + [dev-dependencies.rand] + version = "~0.6" + + [dev-dependencies.rand_pcg] + version = "~0.1" + + :param toml: toml object + """ + + name: str + """The package name""" + + version: str + """Package version""" + + authors: List[str] + """Authors""" + + description: str + """Package and release description""" + + homepage: str + """Homepage of the project""" + + license: str + """Package license""" + + repository: str + """Source code repository""" + + +@attr.s +class CratesPackageInfo(BasePackageInfo): + + name = attr.ib(type=str) + """Name of the package""" + + version = attr.ib(type=str) + """Current version""" + + e_metadata: Dict[str, Any] = attr.ib(factory=ExtrinsicPackageMetadata) + """Extrinsic package metadata, common to all versions""" + + e_metadata_version: Dict[str, Any] = attr.ib( + factory=ExtrinsicVersionPackageMetadata + ) + """Extrinsic package metadata specific to a version""" + + i_metadata: Dict[str, Any] = attr.ib(factory=IntrinsicPackageMetadata) + """Intrinsic metadata of the current package version""" + + +def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: + """Extract intrinsic metadata from Cargo.toml file at dir_path. + + Each crate archive has a Cargo.toml at the root of the archive. + + Args: + dir_path: A directory on disk where a Cargo.toml must be present + + Returns: + A dict mapping from toml parser + """ + return toml.load(dir_path / "Cargo.toml") + + +def extract_author(p_info: CratesPackageInfo) -> Person: + """Extract package author from intrinsic metadata and return it as a + `Person` model. + + Args: + p_info: CratesPackageInfo that should contains i_metadata entries + + Returns: + Only one author (Person) of the package. Currently limited by internal detail + of the swh stack (see T3887). + """ + authors = p_info.i_metadata["authors"] + fullname = authors[0] # TODO: here we have a list of author, see T3887 + return Person.from_fullname(fullname.encode()) + + +def extract_description(p_info: CratesPackageInfo) -> str: + """Extract package description from intrinsic metadata and return it as a + string. + + Args: + p_info: CratesPackageInfo that should contains i_metadata and entries + + Returns: + Package description from metadata. + """ + return p_info.i_metadata["description"] + + +class CratesLoader(PackageLoader[CratesPackageInfo]): + """Load Crates package origins into swh archive.""" + + visit_type = "crates" + + def __init__( + self, + storage: StorageInterface, + url: str, + package_name: str, + version: str, + checksum: Optional[str] = None, + max_content_size: Optional[int] = None, + ): + """Constructor + + Args: + + url : str + Origin url (e.g. + https://static.crates.io/crates//-.crate) + + package_name : str + Crate package name + + version : str + Crate package version + + checksum : str, optional + Checksum for the package file to download + """ + super().__init__(storage=storage, url=url, max_content_size=max_content_size) + self.name = package_name + self.provider_url = f"https://crates.io/api/v1/crates/{self.name}" + # Check consistency of name, version, url + filename = urlparse(url).path.split("/")[-1] + assert f"{self.name}-{version}.crate" == filename + + @cached_method + def _raw_info(self) -> bytes: + """Get crate metadata (fetched from http api endpoint set as self.provider_url) + + Returns: + Content response as bytes. Content response is a json document. + """ + return api_info(self.provider_url) + + @cached_method + def info(self) -> Dict: + """Parse http api json response and return the crate metadata information + as a Dict.""" + return json.loads(self._raw_info()) + + def get_versions(self) -> Sequence[str]: + """Get all released versions of a crate + + Returns: + A sequence of versions + + Example:: + + ["0.1.1", "0.10.2"] + """ + versions = [version["num"] for version in self.info()["versions"]] + versions.sort(key=StrictVersion) + return versions + + def get_default_version(self) -> str: + """Get the newest release version of a crate + + Returns: + A string representing a version + + Example:: + + "0.1.2" + """ + return self.info()["crate"]["newest_version"] + + def get_package_info(self, version: str) -> Iterator[Tuple[str, CratesPackageInfo]]: + """Get release name and package information from version + + Args: + version: crate version (e.g: "0.1.0") + + Returns: + Iterator of tuple (release_name, p_info) + """ + filename = f"{self.name}-{version}.crate" + url = f"https://static.crates.io/crates/{self.name}/{self.name}-{version}.crate" + + # Get extrinsic metadata from http api + + # Raw crate info + e_metadata = ExtrinsicPackageMetadata(**self.info()) # type: ignore[misc] + # Extract crate info for current version (One .crate file for a given version) + (crate_version,) = [ + crate for crate in e_metadata["versions"] if crate["num"] == version + ] + e_metadata_version = ExtrinsicVersionPackageMetadata( # type: ignore[misc] + **crate_version + ) + + p_info = CratesPackageInfo( + name=self.name, + filename=filename, + url=url, + version=version, + e_metadata=e_metadata, + e_metadata_version=e_metadata_version, + ) + yield release_name(version, filename), p_info + + def build_release( + self, p_info: CratesPackageInfo, uncompressed_path: str, directory: Sha1Git + ) -> Optional[Release]: + # Extract intrinsic metadata from dir_path/Cargo.toml + name = p_info.name + version = p_info.version + dir_path = Path(uncompressed_path, f"{name}-{version}") + i_metadata_raw = extract_intrinsic_metadata(dir_path) + # Get only corresponding key of IntrinsicPackageMetadata + i_metadata_keys = [k for k in IntrinsicPackageMetadata.__annotations__.keys()] + # We use data only from "package" entry + + i_metadata = { + k: v for k, v in i_metadata_raw["package"].items() if k in i_metadata_keys + } + p_info.i_metadata = IntrinsicPackageMetadata(**i_metadata) # type: ignore[misc] + + author = extract_author(p_info) + description = extract_description(p_info) + message = ( + f"Synthetic release for Crate source package {p_info.name} " + f"version {p_info.version}\n" + f"{description}\n" + ) + # The only way to get a value for updated_at is through extrinsic metadata + updated_at = p_info.e_metadata_version.get("updated_at") + + return Release( + name=version.encode(), + author=author, + date=TimestampWithTimezone.from_iso8601(updated_at), + message=message.encode(), + target_type=ObjectType.DIRECTORY, + target=directory, + synthetic=True, + ) diff --git a/swh/loader/package/crates/tasks.py b/swh/loader/package/crates/tasks.py new file mode 100644 index 0000000..2d774c9 --- /dev/null +++ b/swh/loader/package/crates/tasks.py @@ -0,0 +1,16 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.package.crates.loader import CratesLoader + + +@shared_task(name=__name__ + ".LoadCrates") +def load_crates(*, url=None, package_name: str, version: str, checksum=None): + """Load Rust crate package""" + return CratesLoader.from_configfile( + url=url, package_name=package_name, version=version, checksum=checksum + ).load() diff --git a/swh/loader/package/crates/tests/__init__.py b/swh/loader/package/crates/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/loader/package/crates/tests/data/fake_crates.sh b/swh/loader/package/crates/tests/data/fake_crates.sh new file mode 100644 index 0000000..9bfff25 --- /dev/null +++ b/swh/loader/package/crates/tests/data/fake_crates.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash + +# Script to generate fake crates files and fake http api response. + +set -euo pipefail + +# files and directories +mkdir -p tmp_dir/crates/ +mkdir tmp_dir/crates/hg-core-0.0.1 +mkdir tmp_dir/crates/micro-timer-0.1.0 +mkdir tmp_dir/crates/micro-timer-0.1.1 +mkdir tmp_dir/crates/micro-timer-0.1.2 +mkdir tmp_dir/crates/micro-timer-0.2.0 +mkdir tmp_dir/crates/micro-timer-0.2.1 +mkdir tmp_dir/crates/micro-timer-0.3.0 +mkdir tmp_dir/crates/micro-timer-0.3.1 +mkdir tmp_dir/crates/micro-timer-0.4.0 + + +cd tmp_dir/crates/ + +# Creates some -.crate file for test purposes. + +# hg-core-0.0.1/Cargo.toml +echo -e '''[package] +name = "hg-core" +version = "0.0.1" +authors = ["Georges Racinet "] +description = "Mercurial pure Rust core library, with no assumption on Python bindings (FFI)" +homepage = "https://mercurial-scm.org" +license = "GPL-2.0-or-later" +repository = "https://www.mercurial-scm.org/repo/hg" + +[lib] +name = "hg" +[dev-dependencies.rand] +version = "~0.6" + +[dev-dependencies.rand_pcg] +version = "~0.1" +''' > hg-core-0.0.1/Cargo.toml + +# micro-timer-0.1.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.1.0/Cargo.toml + +# micro-timer-0.1.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.1.1/Cargo.toml + +# micro-timer-0.1.2/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.2" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.proc-macro2] +version = "1.0.9" + +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" + +[dev-dependencies.pretty_assertions] +version = "0.6.1" +''' > micro-timer-0.1.2/Cargo.toml + +# micro-timer-0.2.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.2.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.2.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.2.0/Cargo.toml + +# micro-timer-0.2.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.2.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.2.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.2.1/Cargo.toml + +# micro-timer-0.3.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.3.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.3.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.3.0/Cargo.toml + +# micro-timer-0.3.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.3.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://foss.heptapod.net/octobus/rust/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://foss.heptapod.net/octobus/rust/micro-timer" +[dependencies.micro-timer-macros] +version = "0.3.1" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.3.1/Cargo.toml + +# micro-timer-0.4.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.4.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://foss.heptapod.net/octobus/rust/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://foss.heptapod.net/octobus/rust/micro-timer" +[dependencies.micro-timer-macros] +version = "0.4.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.4.0/Cargo.toml + +# .crate file are tar.gz archive +tar -czf hg-core-0.0.1.crate hg-core-0.0.1/ +tar -czf micro-timer-0.1.0.crate micro-timer-0.1.0/ +tar -czf micro-timer-0.1.1.crate micro-timer-0.1.1/ +tar -czf micro-timer-0.1.2.crate micro-timer-0.1.2/ +tar -czf micro-timer-0.2.0.crate micro-timer-0.2.0/ +tar -czf micro-timer-0.2.1.crate micro-timer-0.2.1/ +tar -czf micro-timer-0.3.0.crate micro-timer-0.3.0/ +tar -czf micro-timer-0.3.1.crate micro-timer-0.3.1/ +tar -czf micro-timer-0.4.0.crate micro-timer-0.4.0/ + +# Copy and rename .crate file for usage with 'requests_mock_datadir' +# See : https://docs.softwareheritage.org/devel/apidoc/swh.core.pytest_plugin.html#swh.core.pytest_plugin.requests_mock_datadir +mkdir ../../https_static.crates.io + +cp hg-core-0.0.1.crate ../../https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate +cp micro-timer-0.1.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.0.crate +cp micro-timer-0.1.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.1.crate +cp micro-timer-0.1.2.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.2.crate +cp micro-timer-0.2.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.2.0.crate +cp micro-timer-0.2.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.2.1.crate +cp micro-timer-0.3.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.3.0.crate +cp micro-timer-0.3.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate +cp micro-timer-0.4.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate + +# Creates some http file response for test purposes. +mkdir ../../https_crates.io + +# hg-core, https://crates.io/api/v1/crates/hg-core +echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]} +''' > ../../https_crates.io/api_v1_crates_hg-core + +# micro-timer, https://crates.io/api/v1/crates/micro-timer +echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]} +''' > ../../https_crates.io/api_v1_crates_micro-timer + +# Clean up removing tmp_dir +cd ../../ +rm -r tmp_dir/ diff --git a/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core new file mode 100644 index 0000000..c76874d --- /dev/null +++ b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core @@ -0,0 +1,2 @@ +{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]} + diff --git a/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer new file mode 100644 index 0000000..e6878b6 --- /dev/null +++ b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer @@ -0,0 +1,2 @@ +{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]} + diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate new file mode 100644 index 0000000..e51e437 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.0.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.0.crate new file mode 100644 index 0000000..dffc2fc Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.0.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.1.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.1.crate new file mode 100644 index 0000000..6cc3eb6 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.1.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.2.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.2.crate new file mode 100644 index 0000000..3a06370 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.1.2.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.0.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.0.crate new file mode 100644 index 0000000..c304d04 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.0.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.1.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.1.crate new file mode 100644 index 0000000..3d844f0 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.2.1.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.0.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.0.crate new file mode 100644 index 0000000..ff91227 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.0.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate new file mode 100644 index 0000000..fcbe142 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate differ diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate new file mode 100644 index 0000000..1b50fa5 Binary files /dev/null and b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate differ diff --git a/swh/loader/package/crates/tests/test_crates.py b/swh/loader/package/crates/tests/test_crates.py new file mode 100644 index 0000000..e577462 --- /dev/null +++ b/swh/loader/package/crates/tests/test_crates.py @@ -0,0 +1,214 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information +from swh.loader.package.crates.loader import CratesLoader +from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats +from swh.model.hashutil import hash_to_bytes +from swh.model.model import ( + ObjectType, + Person, + Release, + Snapshot, + SnapshotBranch, + TargetType, + TimestampWithTimezone, +) + +CRATES_EXTRA = [ + { + "name": "hg-core", + "version": "0.0.1", + "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate", + "checksum": "7fe168efadebadb9da6a329fdc027036e233b662285730cad27220e11e53c384", + }, + { + "name": "micro-timer", + "version": "0.4.0", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate", + "checksum": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", + }, +] + + +def test_get_versions(requests_mock_datadir, swh_storage): + loader = CratesLoader( + swh_storage, + url=CRATES_EXTRA[1]["url"], + package_name=CRATES_EXTRA[1]["name"], + version=CRATES_EXTRA[1]["version"], + ) + assert loader.get_versions() == [ + "0.1.0", + "0.1.1", + "0.1.2", + "0.2.0", + "0.2.1", + "0.3.0", + "0.3.1", + "0.4.0", + ] + + +def test_get_default_version(requests_mock_datadir, swh_storage): + loader = CratesLoader( + swh_storage, + url=CRATES_EXTRA[1]["url"], + package_name=CRATES_EXTRA[1]["name"], + version=CRATES_EXTRA[1]["version"], + ) + assert loader.get_default_version() == "0.4.0" + + +def test_crate_origin_not_found(swh_storage, requests_mock_datadir): + url = "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate" + loader = CratesLoader( + swh_storage, + url, + package_name="nowhere-to-hide", + version="0.0.1", + ) + + assert loader.load() == {"status": "failed"} + + assert_last_visit_matches( + swh_storage, url, status="not_found", type="crates", snapshot=None + ) + + +def test_crates_loader_load_one_version(datadir, requests_mock_datadir, swh_storage): + loader = CratesLoader( + swh_storage, + url=CRATES_EXTRA[0]["url"], + package_name=CRATES_EXTRA[0]["name"], + version=CRATES_EXTRA[0]["version"], + ) + actual_load_status = loader.load() + assert actual_load_status["status"] == "eventful" + assert actual_load_status["snapshot_id"] is not None + + expected_snapshot_id = "353cd6858c88ee8210432ea1098993c2e9966561" + expected_release_id = "d578833534017430f1b93eb741620899620c2505" + + assert expected_snapshot_id == actual_load_status["snapshot_id"] + + expected_snapshot = Snapshot( + id=hash_to_bytes(actual_load_status["snapshot_id"]), + branches={ + b"releases/0.0.1/hg-core-0.0.1.crate": SnapshotBranch( + target=hash_to_bytes(expected_release_id), + target_type=TargetType.RELEASE, + ), + b"HEAD": SnapshotBranch( + target=b"releases/0.0.1/hg-core-0.0.1.crate", + target_type=TargetType.ALIAS, + ), + }, + ) + check_snapshot(expected_snapshot, swh_storage) + + stats = get_stats(swh_storage) + assert { + "content": 1, + "directory": 2, + "origin": 1, + "origin_visit": 1, + "release": 1, + "revision": 0, + "skipped_content": 0, + "snapshot": 1, + } == stats + + assert swh_storage.release_get([hash_to_bytes(expected_release_id)])[0] == Release( + name=b"0.0.1", + message=b"Synthetic release for Crate source package hg-core version " + b"0.0.1\nMercurial pure Rust core library, with no assumption " + b"on Python bindings (FFI)\n", + target=hash_to_bytes("674c3b0b54628d55b93a79dc7adf304efc01b371"), + target_type=ObjectType.DIRECTORY, + synthetic=True, + author=Person.from_fullname(b"Georges Racinet "), + date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), + id=hash_to_bytes(expected_release_id), + ) + + +def test_crates_loader_load_n_versions(datadir, requests_mock_datadir, swh_storage): + url = CRATES_EXTRA[1]["url"] + loader = CratesLoader( + swh_storage, + url=url, + package_name=CRATES_EXTRA[1]["name"], + version=CRATES_EXTRA[1]["version"], + checksum=CRATES_EXTRA[1]["checksum"], + ) + actual_load_status = loader.load() + assert actual_load_status["status"] == "eventful" + assert actual_load_status["snapshot_id"] is not None + + expected_snapshot_id = "016cbbe3bb78424c35b898015a2d80d79359e2ad" + assert expected_snapshot_id == actual_load_status["snapshot_id"] + + expected_snapshot = Snapshot( + id=hash_to_bytes(expected_snapshot_id), + branches={ + b"releases/0.4.0/micro-timer-0.4.0.crate": SnapshotBranch( + target=hash_to_bytes("3237c1174c4ccfa8e934d1bfd8d80b3a89760e39"), + target_type=TargetType.RELEASE, + ), + b"releases/0.3.1/micro-timer-0.3.1.crate": SnapshotBranch( + target=hash_to_bytes("8b727a280051cdb90468ede2746e176e6fdf355f"), + target_type=TargetType.RELEASE, + ), + b"releases/0.3.0/micro-timer-0.3.0.crate": SnapshotBranch( + target=hash_to_bytes("f45ec236ae50fb37e924a3d2cc093e72b6cbf1cd"), + target_type=TargetType.RELEASE, + ), + b"releases/0.2.1/micro-timer-0.2.1.crate": SnapshotBranch( + target=hash_to_bytes("50a60a2c3696df7cd1b623bd7dbea2c89b994e42"), + target_type=TargetType.RELEASE, + ), + b"releases/0.2.0/micro-timer-0.2.0.crate": SnapshotBranch( + target=hash_to_bytes("f0592dc0ae05399d872017d0260c45b875cb590e"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.2/micro-timer-0.1.2.crate": SnapshotBranch( + target=hash_to_bytes("9220d7823fc40ab44e3ae3227522e7de672fad3e"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch( + target=hash_to_bytes("38529b7e355f79fdce31a3ba891e146174e10237"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch( + target=hash_to_bytes("5e5e6120af55b65c577e09331df54e70fad5e8b0"), + target_type=TargetType.RELEASE, + ), + b"HEAD": SnapshotBranch( + target=b"releases/0.4.0/micro-timer-0.4.0.crate", + target_type=TargetType.ALIAS, + ), + }, + ) + + check_snapshot(expected_snapshot, swh_storage) + + stats = get_stats(swh_storage) + assert { + "content": 8, + "directory": 16, + "origin": 1, + "origin_visit": 1, + "release": 8, + "revision": 0, + "skipped_content": 0, + "snapshot": 1, + } == stats + + assert_last_visit_matches( + swh_storage, + url, + status="full", + type="crates", + snapshot=expected_snapshot.id, + ) diff --git a/swh/loader/package/crates/tests/test_tasks.py b/swh/loader/package/crates/tests/test_tasks.py new file mode 100644 index 0000000..b38412f --- /dev/null +++ b/swh/loader/package/crates/tests/test_tasks.py @@ -0,0 +1,25 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_tasks_crates_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + mock_load = mocker.patch("swh.loader.package.crates.loader.CratesLoader.load") + mock_load.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.crates.tasks.LoadCrates", + kwargs=dict( + url="some-url/crates/some-package/some-package-0.0.1.crate", + package_name="some-package", + version="0.0.1", + ), + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}