diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,6 +18,7 @@ - id: codespell name: Check source code spelling exclude: ^(swh/loader/package/.*[/]+tests/data/.*)$ + args: [-L crate] entry: codespell --ignore-words-list=iff stages: [commit] diff --git a/docs/package-loader-specifications.rst b/docs/package-loader-specifications.rst --- a/docs/package-loader-specifications.rst +++ b/docs/package-loader-specifications.rst @@ -38,6 +38,15 @@ - ``metadata.get(​"Maintainer", "")`` - ``metadata.get(​"Date")`` - metadata is intrinsic + * - crates + - passed as arg + - ``release_name(​version, filename)`` + - =version + - Synthetic release for Crate source package {p_info.name} version {p_info.version} {description} + - true + - from intrinsic metadata + - from extrinsic metadata + - ``i_metadata`` for intrinsic metadata, ``e_metadata`` for extrinsic metadata * - debian - =``version`` - ``release_name(​version)`` diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ python-debian python-dateutil typing-extensions +toml diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -57,6 +57,7 @@ [swh.workers] loader.archive=swh.loader.package.archive:register loader.cran=swh.loader.package.cran:register + loader.crates=swh.loader.package.crates:register loader.debian=swh.loader.package.debian:register loader.deposit=swh.loader.package.deposit:register loader.nixguix=swh.loader.package.nixguix:register diff --git a/swh/loader/package/crates/__init__.py b/swh/loader/package/crates/__init__.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from .loader import CratesLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": CratesLoader, + } diff --git a/swh/loader/package/crates/loader.py b/swh/loader/package/crates/loader.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/loader.py @@ -0,0 +1,355 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from distutils.version import StrictVersion +import json +from pathlib import Path +from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple +from urllib.parse import urlparse + +import attr +import toml +from typing_extensions import TypedDict + +from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.loader.package.utils import api_info, cached_method, release_name +from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone +from swh.storage.interface import StorageInterface + + +class ExtrinsicPackageMetadata(TypedDict): + """Data structure for package extrinsic metadata pulled from http api endpoint. + + We set only the keys we need according to what is available when querying + https://crates.io/api/v1/crates/, where `name` is the name of the crate + package. + + Json response example for https://crates.io/api/v1/crates/hg-core : + + .. literalinclude:: ../../swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core + + Usage example: + + .. code-block:: python + + e_metadata = ExtrinsicPackageMetadata(**self.info()) + + """ # noqa + + categories: List[Dict[Any, Any]] + """Related categories""" + + crate: Dict[Any, Any] + """Crate project information""" + + keywords: List[Any] + """Keywords""" + + versions: List[Dict[Any, Any]] + """A list of released versions for a crate""" + + +class ExtrinsicVersionPackageMetadata(TypedDict): + """Data structure for specific package version extrinsic metadata, pulled + from http api endpoint. + + Similar to `ExtrinsicPackageMetadata` in its usage, but we flatten the data + related to a specific version. + """ + + crate: str + """The package name""" + + crate_size: int + """The package size""" + + created_at: str + """First released at""" + + downloads: str + """Number of downloads""" + + license: str + """Package license""" + + num: str + """Package version""" + + published_by: Dict[Any, Any] + """Publishers information""" + + updated_at: str + """Last update""" + + yanked: bool + """Is that version yanked? (yanked means release-level deprecation)""" + + +class IntrinsicPackageMetadata(TypedDict): + """Data structure for specific package version intrinsic metadata. + + Data is extracted from the crate package's .toml file. Then the data of the + 'package' entry is flattened. + + Cargo.toml file content example: + + .. code-block:: toml + + [package] + name = "hg-core" + version = "0.0.1" + authors = ["Georges Racinet "] + description = "Mercurial pure Rust core library, with no assumption on + Python bindings (FFI)" + homepage = "https://mercurial-scm.org" + license = "GPL-2.0-or-later" + repository = "https://www.mercurial-scm.org/repo/hg" + + [lib] + name = "hg" + [dev-dependencies.rand] + version = "~0.6" + + [dev-dependencies.rand_pcg] + version = "~0.1" + + :param toml: toml object + """ + + name: str + """The package name""" + + version: str + """Package version""" + + authors: List[str] + """Authors""" + + description: str + """Package and release description""" + + homepage: str + """Homepage of the project""" + + license: str + """Package license""" + + repository: str + """Source code repository""" + + +@attr.s +class CratesPackageInfo(BasePackageInfo): + + name = attr.ib(type=str) + """Name of the package""" + + version = attr.ib(type=str) + """Current version""" + + e_metadata: Dict[str, Any] = attr.ib(factory=ExtrinsicPackageMetadata) + """Extrinsic package metadata, common to all versions""" + + e_metadata_version: Dict[str, Any] = attr.ib( + factory=ExtrinsicVersionPackageMetadata + ) + """Extrinsic package metadata specific to a version""" + + i_metadata: Dict[str, Any] = attr.ib(factory=IntrinsicPackageMetadata) + """Intrinsic metadata of the current package version""" + + +def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: + """Extract intrinsic metadata from Cargo.toml file at dir_path. + + Each crate archive has a Cargo.toml at the root of the archive. + + Args: + dir_path: A directory on disk where a Cargo.toml must be present + + Returns: + A dict mapping from toml parser + """ + return toml.load(dir_path / "Cargo.toml") + + +def extract_author(p_info: CratesPackageInfo) -> Person: + """Extract package author from intrinsic metadata and return it as a + `Person` model. + + Args: + p_info: CratesPackageInfo that should contains i_metadata entries + + Returns: + Only one author (Person) of the package. Currently limited by internal detail + of the swh stack (see T3887). + """ + authors = p_info.i_metadata["authors"] + fullname = authors[0] # TODO: here we have a list of author, see T3887 + return Person.from_fullname(fullname.encode()) + + +def extract_description(p_info: CratesPackageInfo) -> str: + """Extract package description from intrinsic metadata and return it as a + string. + + Args: + p_info: CratesPackageInfo that should contains i_metadata and entries + + Returns: + Package description from metadata. + """ + return p_info.i_metadata["description"] + + +class CratesLoader(PackageLoader[CratesPackageInfo]): + """Load Crates package origins into swh archive.""" + + visit_type = "crates" + + def __init__( + self, + storage: StorageInterface, + url: str, + package_name: str, + version: str, + checksum: Optional[str] = None, + max_content_size: Optional[int] = None, + ): + """Constructor + + Args: + + url : str + Origin url (e.g. + https://static.crates.io/crates//-.crate) + + package_name : str + Crate package name + + version : str + Crate package version + + checksum : str, optional + Checksum for the package file to download + """ + super().__init__(storage=storage, url=url, max_content_size=max_content_size) + self.name = package_name + self.provider_url = f"https://crates.io/api/v1/crates/{self.name}" + # Check consistency of name, version, url + filename = urlparse(url).path.split("/")[-1] + assert f"{self.name}-{version}.crate" == filename + + @cached_method + def _raw_info(self) -> bytes: + """Get crate metadata (fetched from http api endpoint set as self.provider_url) + + Returns: + Content response as bytes. Content response is a json document. + """ + return api_info(self.provider_url) + + @cached_method + def info(self) -> Dict: + """Parse http api json response and return the crate metadata information + as a Dict.""" + return json.loads(self._raw_info()) + + def get_versions(self) -> Sequence[str]: + """Get all released versions of a crate + + Returns: + A sequence of versions + + Example:: + + ["0.1.1", "0.10.2"] + """ + versions = [version["num"] for version in self.info()["versions"]] + versions.sort(key=StrictVersion) + return versions + + def get_default_version(self) -> str: + """Get the newest release version of a crate + + Returns: + A string representing a version + + Example:: + + "0.1.2" + """ + return self.info()["crate"]["newest_version"] + + def get_package_info(self, version: str) -> Iterator[Tuple[str, CratesPackageInfo]]: + """Get release name and package information from version + + Args: + version: crate version (e.g: "0.1.0") + + Returns: + Iterator of tuple (release_name, p_info) + """ + filename = f"{self.name}-{version}.crate" + url = f"https://static.crates.io/crates/{self.name}/{self.name}-{version}.crate" + + # Get extrinsic metadata from http api + + # Raw crate info + e_metadata = ExtrinsicPackageMetadata(**self.info()) # type: ignore[misc] + # Extract crate info for current version (One .crate file for a given version) + (crate_version,) = [ + crate for crate in e_metadata["versions"] if crate["num"] == version + ] + e_metadata_version = ExtrinsicVersionPackageMetadata( # type: ignore[misc] + **crate_version + ) + + p_info = CratesPackageInfo( + name=self.name, + filename=filename, + url=url, + version=version, + e_metadata=e_metadata, + e_metadata_version=e_metadata_version, + ) + yield release_name(version, filename), p_info + + def build_release( + self, p_info: CratesPackageInfo, uncompressed_path: str, directory: Sha1Git + ) -> Optional[Release]: + # Extract intrinsic metadata from dir_path/Cargo.toml + name = p_info.name + version = p_info.version + dir_path = Path(uncompressed_path, f"{name}-{version}") + i_metadata_raw = extract_intrinsic_metadata(dir_path) + # Get only corresponding key of IntrinsicPackageMetadata + i_metadata_keys = [k for k in IntrinsicPackageMetadata.__annotations__.keys()] + # We use data only from "package" entry + + i_metadata = { + k: v for k, v in i_metadata_raw["package"].items() if k in i_metadata_keys + } + p_info.i_metadata = IntrinsicPackageMetadata(**i_metadata) # type: ignore[misc] + + author = extract_author(p_info) + description = extract_description(p_info) + message = ( + f"Synthetic release for Crate source package {p_info.name} " + f"version {p_info.version}\n" + f"{description}\n" + ) + # The only way to get a value for updated_at is through extrinsic metadata + updated_at = p_info.e_metadata_version.get("updated_at") + + return Release( + name=version.encode(), + author=author, + date=TimestampWithTimezone.from_iso8601(updated_at), + message=message.encode(), + target_type=ObjectType.DIRECTORY, + target=directory, + synthetic=True, + ) diff --git a/swh/loader/package/crates/tasks.py b/swh/loader/package/crates/tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/tasks.py @@ -0,0 +1,16 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.package.crates.loader import CratesLoader + + +@shared_task(name=__name__ + ".LoadCrates") +def load_crates(*, url=None, package_name: str, version: str, checksum=None): + """Load Rust crate package""" + return CratesLoader.from_configfile( + url=url, package_name=package_name, version=version, checksum=checksum + ).load() diff --git a/swh/loader/package/crates/tests/__init__.py b/swh/loader/package/crates/tests/__init__.py new file mode 100644 diff --git a/swh/loader/package/crates/tests/data/fake_crates.sh b/swh/loader/package/crates/tests/data/fake_crates.sh new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/tests/data/fake_crates.sh @@ -0,0 +1,259 @@ +#!/usr/bin/env bash + +# Script to generate fake crates files and fake http api response. + +set -euo pipefail + +# files and directories +mkdir -p tmp_dir/crates/ +mkdir tmp_dir/crates/hg-core-0.0.1 +mkdir tmp_dir/crates/micro-timer-0.1.0 +mkdir tmp_dir/crates/micro-timer-0.1.1 +mkdir tmp_dir/crates/micro-timer-0.1.2 +mkdir tmp_dir/crates/micro-timer-0.2.0 +mkdir tmp_dir/crates/micro-timer-0.2.1 +mkdir tmp_dir/crates/micro-timer-0.3.0 +mkdir tmp_dir/crates/micro-timer-0.3.1 +mkdir tmp_dir/crates/micro-timer-0.4.0 + + +cd tmp_dir/crates/ + +# Creates some -.crate file for test purposes. + +# hg-core-0.0.1/Cargo.toml +echo -e '''[package] +name = "hg-core" +version = "0.0.1" +authors = ["Georges Racinet "] +description = "Mercurial pure Rust core library, with no assumption on Python bindings (FFI)" +homepage = "https://mercurial-scm.org" +license = "GPL-2.0-or-later" +repository = "https://www.mercurial-scm.org/repo/hg" + +[lib] +name = "hg" +[dev-dependencies.rand] +version = "~0.6" + +[dev-dependencies.rand_pcg] +version = "~0.1" +''' > hg-core-0.0.1/Cargo.toml + +# micro-timer-0.1.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.1.0/Cargo.toml + +# micro-timer-0.1.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.1.1/Cargo.toml + +# micro-timer-0.1.2/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.1.2" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" + +[lib] +proc-macro = true +[dependencies.proc-macro2] +version = "1.0.9" + +[dependencies.quote] +version = "1.0.2" + +[dependencies.syn] +version = "1.0.16" +features = ["full", "extra-traits"] +[dev-dependencies.log] +version = "0.4.8" + +[dev-dependencies.pretty_assertions] +version = "0.6.1" +''' > micro-timer-0.1.2/Cargo.toml + +# micro-timer-0.2.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.2.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.2.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.2.0/Cargo.toml + +# micro-timer-0.2.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.2.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.2.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.2.1/Cargo.toml + +# micro-timer-0.3.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.3.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://heptapod.octobus.net/Alphare/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://heptapod.octobus.net/Alphare/micro-timer" +[dependencies.micro-timer-macros] +version = "0.3.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.3.0/Cargo.toml + +# micro-timer-0.3.1/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.3.1" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://foss.heptapod.net/octobus/rust/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://foss.heptapod.net/octobus/rust/micro-timer" +[dependencies.micro-timer-macros] +version = "0.3.1" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.3.1/Cargo.toml + +# micro-timer-0.4.0/Cargo.toml +echo -e '''[package] +edition = "2018" +name = "micro-timer" +version = "0.4.0" +authors = ["Raphaël Gomès "] +description = "Dumb tiny logging timer" +homepage = "https://foss.heptapod.net/octobus/rust/micro-timer" +readme = "README.md" +license-file = "LICENCE" +repository = "https://foss.heptapod.net/octobus/rust/micro-timer" +[dependencies.micro-timer-macros] +version = "0.4.0" + +[dependencies.scopeguard] +version = "1.1.0" +[dev-dependencies.log] +version = "0.4.8" +''' > micro-timer-0.4.0/Cargo.toml + +# .crate file are tar.gz archive +tar -czf hg-core-0.0.1.crate hg-core-0.0.1/ +tar -czf micro-timer-0.1.0.crate micro-timer-0.1.0/ +tar -czf micro-timer-0.1.1.crate micro-timer-0.1.1/ +tar -czf micro-timer-0.1.2.crate micro-timer-0.1.2/ +tar -czf micro-timer-0.2.0.crate micro-timer-0.2.0/ +tar -czf micro-timer-0.2.1.crate micro-timer-0.2.1/ +tar -czf micro-timer-0.3.0.crate micro-timer-0.3.0/ +tar -czf micro-timer-0.3.1.crate micro-timer-0.3.1/ +tar -czf micro-timer-0.4.0.crate micro-timer-0.4.0/ + +# Copy and rename .crate file for usage with 'requests_mock_datadir' +# See : https://docs.softwareheritage.org/devel/apidoc/swh.core.pytest_plugin.html#swh.core.pytest_plugin.requests_mock_datadir +mkdir ../../https_static.crates.io + +cp hg-core-0.0.1.crate ../../https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate +cp micro-timer-0.1.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.0.crate +cp micro-timer-0.1.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.1.crate +cp micro-timer-0.1.2.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.1.2.crate +cp micro-timer-0.2.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.2.0.crate +cp micro-timer-0.2.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.2.1.crate +cp micro-timer-0.3.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.3.0.crate +cp micro-timer-0.3.1.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.3.1.crate +cp micro-timer-0.4.0.crate ../../https_static.crates.io/crates_micro-timer_micro-timer-0.4.0.crate + +# Creates some http file response for test purposes. +mkdir ../../https_crates.io + +# hg-core, https://crates.io/api/v1/crates/hg-core +echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]} +''' > ../../https_crates.io/api_v1_crates_hg-core + +# micro-timer, https://crates.io/api/v1/crates/micro-timer +echo -e '''{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]} +''' > ../../https_crates.io/api_v1_crates_micro-timer + +# Clean up removing tmp_dir +cd ../../ +rm -r tmp_dir/ diff --git a/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_hg-core @@ -0,0 +1,2 @@ +{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2019-04-16T18:48:11.404457+00:00","description":"Mercurial pure Rust core library, with no assumption on Python bindings (FFI)","documentation":null,"downloads":442,"exact_match":false,"homepage":"https://mercurial-scm.org","id":"hg-core","keywords":[],"links":{"owner_team":"/api/v1/crates/hg-core/owner_team","owner_user":"/api/v1/crates/hg-core/owner_user","owners":"/api/v1/crates/hg-core/owners","reverse_dependencies":"/api/v1/crates/hg-core/reverse_dependencies","version_downloads":"/api/v1/crates/hg-core/downloads","versions":null},"max_stable_version":"0.0.1","max_version":"0.0.1","name":"hg-core","newest_version":"0.0.1","recent_downloads":40,"repository":"https://www.mercurial-scm.org/repo/hg","updated_at":"2019-04-16T18:48:11.404457+00:00","versions":[145309]},"keywords":[],"versions":[{"audit_actions":[],"crate":"hg-core","crate_size":21344,"created_at":"2019-04-16T18:48:11.404457+00:00","dl_path":"/api/v1/crates/hg-core/0.0.1/download","downloads":442,"features":{},"id":145309,"license":"GPL-2.0-or-later","links":{"authors":"/api/v1/crates/hg-core/0.0.1/authors","dependencies":"/api/v1/crates/hg-core/0.0.1/dependencies","version_downloads":"/api/v1/crates/hg-core/0.0.1/downloads"},"num":"0.0.1","published_by":{"avatar":"https://avatars0.githubusercontent.com/u/474220?v=4","id":45544,"login":"gracinet","name":"Georges Racinet","url":"https://github.com/gracinet"},"readme_path":"/api/v1/crates/hg-core/0.0.1/readme","updated_at":"2019-04-16T18:48:11.404457+00:00","yanked":false}]} + diff --git a/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/tests/data/https_crates.io/api_v1_crates_micro-timer @@ -0,0 +1,2 @@ +{"categories":[],"crate":{"badges":[],"categories":[],"created_at":"2020-02-27T14:31:49.131258+00:00","description":"Dumb tiny logging timer","documentation":null,"downloads":44245,"exact_match":false,"homepage":"https://foss.heptapod.net/octobus/rust/micro-timer","id":"micro-timer","keywords":[],"links":{"owner_team":"/api/v1/crates/micro-timer/owner_team","owner_user":"/api/v1/crates/micro-timer/owner_user","owners":"/api/v1/crates/micro-timer/owners","reverse_dependencies":"/api/v1/crates/micro-timer/reverse_dependencies","version_downloads":"/api/v1/crates/micro-timer/downloads","versions":null},"max_stable_version":"0.4.0","max_version":"0.4.0","name":"micro-timer","newest_version":"0.4.0","recent_downloads":3910,"repository":"https://foss.heptapod.net/octobus/rust/micro-timer","updated_at":"2020-09-28T13:40:49.593030+00:00","versions":[288167,254896,248120,223660,223652,216405,216156,216139]},"keywords":[],"versions":[{"audit_actions":[{"action":"publish","time":"2020-09-28T13:40:49.593030+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3513,"created_at":"2020-09-28T13:40:49.593030+00:00","dl_path":"/api/v1/crates/micro-timer/0.4.0/download","downloads":337,"features":{},"id":288167,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.4.0/authors","dependencies":"/api/v1/crates/micro-timer/0.4.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.4.0/downloads"},"num":"0.4.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.4.0/readme","updated_at":"2020-09-28T13:40:49.593030+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-22T16:40:06.754009+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3357,"created_at":"2020-06-22T16:40:06.754009+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.1/download","downloads":37853,"features":{},"id":254896,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.1/authors","dependencies":"/api/v1/crates/micro-timer/0.3.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.1/downloads"},"num":"0.3.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.1/readme","updated_at":"2020-06-22T16:40:06.754009+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-06-02T11:38:33.047581+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":3306,"created_at":"2020-06-02T11:38:33.047581+00:00","dl_path":"/api/v1/crates/micro-timer/0.3.0/download","downloads":4163,"features":{},"id":248120,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.3.0/authors","dependencies":"/api/v1/crates/micro-timer/0.3.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.3.0/downloads"},"num":"0.3.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.3.0/readme","updated_at":"2020-06-02T11:38:33.047581+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T11:22:26.288804+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2937,"created_at":"2020-03-23T11:22:26.288804+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.1/download","downloads":1301,"features":{},"id":223660,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.1/authors","dependencies":"/api/v1/crates/micro-timer/0.2.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.1/downloads"},"num":"0.2.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.1/readme","updated_at":"2020-03-23T11:22:26.288804+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-03-23T10:57:04.418462+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2941,"created_at":"2020-03-23T10:57:04.418462+00:00","dl_path":"/api/v1/crates/micro-timer/0.2.0/download","downloads":104,"features":{},"id":223652,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.2.0/authors","dependencies":"/api/v1/crates/micro-timer/0.2.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.2.0/downloads"},"num":"0.2.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.2.0/readme","updated_at":"2020-03-23T10:57:04.418462+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T23:35:41.872176+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":4927,"created_at":"2020-02-27T23:35:41.872176+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.2/download","downloads":258,"features":{},"id":216405,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.2/authors","dependencies":"/api/v1/crates/micro-timer/0.1.2/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.2/downloads"},"num":"0.1.2","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.2/readme","updated_at":"2020-02-27T23:35:41.872176+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T15:17:53.486346+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2916,"created_at":"2020-02-27T15:17:53.486346+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.1/download","downloads":111,"features":{},"id":216156,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.1/authors","dependencies":"/api/v1/crates/micro-timer/0.1.1/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.1/downloads"},"num":"0.1.1","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.1/readme","updated_at":"2020-02-27T15:17:53.486346+00:00","yanked":false},{"audit_actions":[{"action":"publish","time":"2020-02-27T14:31:49.131258+00:00","user":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"}}],"crate":"micro-timer","crate_size":2902,"created_at":"2020-02-27T14:31:49.131258+00:00","dl_path":"/api/v1/crates/micro-timer/0.1.0/download","downloads":118,"features":{},"id":216139,"license":"non-standard","links":{"authors":"/api/v1/crates/micro-timer/0.1.0/authors","dependencies":"/api/v1/crates/micro-timer/0.1.0/dependencies","version_downloads":"/api/v1/crates/micro-timer/0.1.0/downloads"},"num":"0.1.0","published_by":{"avatar":"https://avatars.githubusercontent.com/u/9445758?v=4","id":79957,"login":"Alphare","name":"Raphaël Gomès","url":"https://github.com/Alphare"},"readme_path":"/api/v1/crates/micro-timer/0.1.0/readme","updated_at":"2020-02-27T14:31:49.131258+00:00","yanked":false}]} + diff --git a/swh/loader/package/crates/tests/data/https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate b/swh/loader/package/crates/tests/data/https_static.crates.io/crates_hg-core_hg-core-0.0.1.crate new file mode 100644 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 GIT binary patch literal 0 Hc$@"), + date=TimestampWithTimezone.from_iso8601("2019-04-16T18:48:11.404457+00:00"), + id=hash_to_bytes(expected_release_id), + ) + + +def test_crates_loader_load_n_versions(datadir, requests_mock_datadir, swh_storage): + url = CRATES_EXTRA[1]["url"] + loader = CratesLoader( + swh_storage, + url=url, + package_name=CRATES_EXTRA[1]["name"], + version=CRATES_EXTRA[1]["version"], + checksum=CRATES_EXTRA[1]["checksum"], + ) + actual_load_status = loader.load() + assert actual_load_status["status"] == "eventful" + assert actual_load_status["snapshot_id"] is not None + + expected_snapshot_id = "016cbbe3bb78424c35b898015a2d80d79359e2ad" + assert expected_snapshot_id == actual_load_status["snapshot_id"] + + expected_snapshot = Snapshot( + id=hash_to_bytes(expected_snapshot_id), + branches={ + b"releases/0.4.0/micro-timer-0.4.0.crate": SnapshotBranch( + target=hash_to_bytes("3237c1174c4ccfa8e934d1bfd8d80b3a89760e39"), + target_type=TargetType.RELEASE, + ), + b"releases/0.3.1/micro-timer-0.3.1.crate": SnapshotBranch( + target=hash_to_bytes("8b727a280051cdb90468ede2746e176e6fdf355f"), + target_type=TargetType.RELEASE, + ), + b"releases/0.3.0/micro-timer-0.3.0.crate": SnapshotBranch( + target=hash_to_bytes("f45ec236ae50fb37e924a3d2cc093e72b6cbf1cd"), + target_type=TargetType.RELEASE, + ), + b"releases/0.2.1/micro-timer-0.2.1.crate": SnapshotBranch( + target=hash_to_bytes("50a60a2c3696df7cd1b623bd7dbea2c89b994e42"), + target_type=TargetType.RELEASE, + ), + b"releases/0.2.0/micro-timer-0.2.0.crate": SnapshotBranch( + target=hash_to_bytes("f0592dc0ae05399d872017d0260c45b875cb590e"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.2/micro-timer-0.1.2.crate": SnapshotBranch( + target=hash_to_bytes("9220d7823fc40ab44e3ae3227522e7de672fad3e"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.1/micro-timer-0.1.1.crate": SnapshotBranch( + target=hash_to_bytes("38529b7e355f79fdce31a3ba891e146174e10237"), + target_type=TargetType.RELEASE, + ), + b"releases/0.1.0/micro-timer-0.1.0.crate": SnapshotBranch( + target=hash_to_bytes("5e5e6120af55b65c577e09331df54e70fad5e8b0"), + target_type=TargetType.RELEASE, + ), + b"HEAD": SnapshotBranch( + target=b"releases/0.4.0/micro-timer-0.4.0.crate", + target_type=TargetType.ALIAS, + ), + }, + ) + + check_snapshot(expected_snapshot, swh_storage) + + stats = get_stats(swh_storage) + assert { + "content": 8, + "directory": 16, + "origin": 1, + "origin_visit": 1, + "release": 8, + "revision": 0, + "skipped_content": 0, + "snapshot": 1, + } == stats + + assert_last_visit_matches( + swh_storage, + url, + status="full", + type="crates", + snapshot=expected_snapshot.id, + ) diff --git a/swh/loader/package/crates/tests/test_tasks.py b/swh/loader/package/crates/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/crates/tests/test_tasks.py @@ -0,0 +1,25 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_tasks_crates_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + mock_load = mocker.patch("swh.loader.package.crates.loader.CratesLoader.load") + mock_load.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.crates.tasks.LoadCrates", + kwargs=dict( + url="some-url/crates/some-package/some-package-0.0.1.crate", + package_name="some-package", + version="0.0.1", + ), + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}