diff --git a/docs/package-loader-specifications.rst b/docs/package-loader-specifications.rst --- a/docs/package-loader-specifications.rst +++ b/docs/package-loader-specifications.rst @@ -39,7 +39,7 @@ - ``metadata.get(​"Date")`` - metadata is intrinsic * - crates - - passed as arg + - ``p_info.​version`` - ``release_name(​version, filename)`` - =version - Synthetic release for Crate source package {p_info.name} version {p_info.version} {description} diff --git a/swh/loader/package/crates/loader.py b/swh/loader/package/crates/loader.py --- a/swh/loader/package/crates/loader.py +++ b/swh/loader/package/crates/loader.py @@ -213,43 +213,45 @@ self, storage: StorageInterface, url: str, - package_name: str, - version: str, - checksum: Optional[str] = None, + artifacts: List[Dict[Any, Any]], max_content_size: Optional[int] = None, ): """Constructor Args: - url : str - Origin url (e.g. - https://static.crates.io/crates//-.crate) + url: + Origin url, (e.g. https://crates.io/api/v1/crates/) - package_name : str - Crate package name + artifacts: + A list of dict listing all existing released versions for a + package (Usually set with crates lister `extra_loader_arguments`). + Each line is a dict that should have an `url` + (where to download package specific version) and a `version` entry. - version : str - Crate package version - checksum : str, optional - Checksum for the package file to download - """ + Example:: + + [ + { + "version": , + "url": "https://static.crates.io/crates//-.crate", + } + ] + """ # noqa + super().__init__(storage=storage, url=url, max_content_size=max_content_size) - self.name = package_name - self.provider_url = f"https://crates.io/api/v1/crates/{self.name}" - # Check consistency of name, version, url - filename = urlparse(url).path.split("/")[-1] - assert f"{self.name}-{version}.crate" == filename + self.url = url + self.artifacts = artifacts @cached_method def _raw_info(self) -> bytes: - """Get crate metadata (fetched from http api endpoint set as self.provider_url) + """Get crate metadata (fetched from http api endpoint set as self.url) Returns: Content response as bytes. Content response is a json document. """ - return api_info(self.provider_url) + return api_info(self.url) @cached_method def info(self) -> Dict: @@ -267,7 +269,7 @@ ["0.1.1", "0.10.2"] """ - versions = [version["num"] for version in self.info()["versions"]] + versions = [item["version"] for item in self.artifacts] versions.sort(key=StrictVersion) return versions @@ -281,7 +283,7 @@ "0.1.2" """ - return self.info()["crate"]["newest_version"] + return self.get_versions()[-1] def get_package_info(self, version: str) -> Iterator[Tuple[str, CratesPackageInfo]]: """Get release name and package information from version @@ -292,8 +294,12 @@ Returns: Iterator of tuple (release_name, p_info) """ - filename = f"{self.name}-{version}.crate" - url = f"https://static.crates.io/crates/{self.name}/{self.name}-{version}.crate" + (artifact,) = [ + artifact for artifact in self.artifacts if artifact["version"] == version + ] + filename = artifact["filename"] + package_name = urlparse(self.url).path.split("/")[-1] + url = artifact["url"] # Get extrinsic metadata from http api @@ -308,7 +314,7 @@ ) p_info = CratesPackageInfo( - name=self.name, + name=package_name, filename=filename, url=url, version=version, @@ -328,7 +334,6 @@ # Get only corresponding key of IntrinsicPackageMetadata i_metadata_keys = [k for k in IntrinsicPackageMetadata.__annotations__.keys()] # We use data only from "package" entry - i_metadata = { k: v for k, v in i_metadata_raw["package"].items() if k in i_metadata_keys } diff --git a/swh/loader/package/crates/tasks.py b/swh/loader/package/crates/tasks.py --- a/swh/loader/package/crates/tasks.py +++ b/swh/loader/package/crates/tasks.py @@ -9,8 +9,6 @@ @shared_task(name=__name__ + ".LoadCrates") -def load_crates(*, url=None, package_name: str, version: str, checksum=None): +def load_crates(*, url=None, artifacts: list): """Load Rust crate package""" - return CratesLoader.from_configfile( - url=url, package_name=package_name, version=version, checksum=checksum - ).load() + return CratesLoader.from_configfile(url=url, artifacts=artifacts).load() diff --git a/swh/loader/package/crates/tests/test_crates.py b/swh/loader/package/crates/tests/test_crates.py --- a/swh/loader/package/crates/tests/test_crates.py +++ b/swh/loader/package/crates/tests/test_crates.py @@ -2,6 +2,8 @@ # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import pytest + from swh.loader.package.crates.loader import CratesLoader from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats from swh.model.hashutil import hash_to_bytes @@ -17,16 +19,86 @@ CRATES_EXTRA = [ { - "name": "hg-core", - "version": "0.0.1", - "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate", - "checksum": "7fe168efadebadb9da6a329fdc027036e233b662285730cad27220e11e53c384", + "url": "https://crates.io/api/v1/crates/hg-core", + "artifacts": [ + { + "checksums": { + "sha256": "48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d", # noqa: B950 + }, + "filename": "hg-core-0.0.1.crate", + "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate", + "version": "0.0.1", + }, + ], }, { - "name": "micro-timer", - "version": "0.4.0", - "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate", - "checksum": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", + "url": "https://crates.io/api/v1/crates/micro-timer", + "artifacts": [ + { + "checksums": { + "sha256": "69ad8fd116f8af0298ae4e83e587b1600af12709022471e25581c3aeb1da77ce", # noqa: B950 + }, + "filename": "micro-timer-0.1.0.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.0.crate", + "version": "0.1.0", + }, + { + "checksums": { + "sha256": "7b3f65fe0e109daad8d47e1938c9b5f9353efacd86bbe7ff013f84ae7ca758bf", # noqa: B950 + }, + "filename": "micro-timer-0.1.1.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.1.crate", + "version": "0.1.1", + }, + { + "checksums": { + "sha256": "16439fea388f712c1df7737ceb8f784d407844624b4796faf1e1bf8bbaa97445", # noqa: B950 + }, + "filename": "micro-timer-0.1.2.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.2.crate", + "version": "0.1.2", + }, + { + "checksums": { + "sha256": "336b4c0f071d16674747faa4643d742cc096fec2bf8cf01bb1a98d984bedcaf1", # noqa: B950 + }, + "filename": "micro-timer-0.2.0.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.0.crate", + "version": "0.2.0", + }, + { + "checksums": { + "sha256": "987429cd6162a80ed5ff44fc790f5090b1c6d617ac73a2e272965ed91201d79b", # noqa: B950 + }, + "filename": "micro-timer-0.2.1.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.1.crate", + "version": "0.2.1", + }, + { + "checksums": { + "sha256": "25b31d6cb9112984323d05d7a353f272ae5d7a307074f9ab9b25c00121b8c947", # noqa: B950 + }, + "filename": "micro-timer-0.3.0.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.0.crate", + "version": "0.3.0", + }, + { + "checksums": { + "sha256": "2620153e1d903d26b72b89f0e9c48d8c4756cba941c185461dddc234980c298c", # noqa: B950 + }, + "filename": "micro-timer-0.3.1.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.1.crate", + "version": "0.3.1", + }, + { + "checksums": { + "sha256": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", # noqa: B950 + }, + "filename": "micro-timer-0.4.0.crate", + "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate", + "version": "0.4.0", + }, + ], }, ] @@ -35,8 +107,7 @@ loader = CratesLoader( swh_storage, url=CRATES_EXTRA[1]["url"], - package_name=CRATES_EXTRA[1]["name"], - version=CRATES_EXTRA[1]["version"], + artifacts=CRATES_EXTRA[1]["artifacts"], ) assert loader.get_versions() == [ "0.1.0", @@ -54,25 +125,44 @@ loader = CratesLoader( swh_storage, url=CRATES_EXTRA[1]["url"], - package_name=CRATES_EXTRA[1]["name"], - version=CRATES_EXTRA[1]["version"], + artifacts=CRATES_EXTRA[1]["artifacts"], ) assert loader.get_default_version() == "0.4.0" -def test_crate_origin_not_found(swh_storage, requests_mock_datadir): - url = "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate" +def test_crate_invalid_origin_archive_not_found(swh_storage, requests_mock_datadir): + url = "https://nowhere-to-run/nowhere-to-hide" loader = CratesLoader( swh_storage, url, - package_name="nowhere-to-hide", - version="0.0.1", + artifacts=[ + { + "filename": "nowhere-to-hide-0.0.1.crate", + "url": "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate", + "version": "0.0.1", + }, + ], ) - assert loader.load() == {"status": "failed"} + with pytest.raises(Exception): + assert loader.load() == {"status": "failed"} + assert_last_visit_matches( + swh_storage, url, status="not_found", type="crates", snapshot=None + ) + +def test_crate_invalid_origin_empty_artifacts(swh_storage, requests_mock_datadir): + url = "https://nowhere-to-run/nowhere-to-hide" + loader = CratesLoader( + swh_storage, + url, + artifacts=[ + {}, + ], + ) + assert loader.load() == {"status": "failed"} assert_last_visit_matches( - swh_storage, url, status="not_found", type="crates", snapshot=None + swh_storage, url, status="failed", type="crates", snapshot=None ) @@ -80,8 +170,7 @@ loader = CratesLoader( swh_storage, url=CRATES_EXTRA[0]["url"], - package_name=CRATES_EXTRA[0]["name"], - version=CRATES_EXTRA[0]["version"], + artifacts=CRATES_EXTRA[0]["artifacts"], ) actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" @@ -137,11 +226,10 @@ url = CRATES_EXTRA[1]["url"] loader = CratesLoader( swh_storage, - url=url, - package_name=CRATES_EXTRA[1]["name"], - version=CRATES_EXTRA[1]["version"], - checksum=CRATES_EXTRA[1]["checksum"], + url=CRATES_EXTRA[1]["url"], + artifacts=CRATES_EXTRA[1]["artifacts"], ) + actual_load_status = loader.load() assert actual_load_status["status"] == "eventful" assert actual_load_status["snapshot_id"] is not None diff --git a/swh/loader/package/crates/tests/test_tasks.py b/swh/loader/package/crates/tests/test_tasks.py --- a/swh/loader/package/crates/tests/test_tasks.py +++ b/swh/loader/package/crates/tests/test_tasks.py @@ -13,9 +13,8 @@ res = swh_scheduler_celery_app.send_task( "swh.loader.package.crates.tasks.LoadCrates", kwargs=dict( - url="some-url/crates/some-package/some-package-0.0.1.crate", - package_name="some-package", - version="0.0.1", + url="some-url/api/v1/crates/some-package", + artifacts=[{"version": "0.0.1", "url": "some-package-0.0.1.crate"}], ), ) assert res