Page MenuHomeSoftware Heritage
Files F9343960

D7713.diff
No OneTemporary
Actions

D7713.diff
View Options

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -1,2 +1,3 @@
Antoine Eiche
Léo Andrès
+Franck Bret
diff --git a/docs/package-loader-specifications.rst b/docs/package-loader-specifications.rst
--- a/docs/package-loader-specifications.rst
+++ b/docs/package-loader-specifications.rst
@@ -39,7 +39,7 @@
- ``metadata.get(​"Date")``
- metadata is intrinsic
* - crates
- - passed as arg
+ - ``p_info.​version``
- ``release_name(​version, filename)``
- =version
- Synthetic release for Crate source package {p_info.name} version {p_info.version} {description}
diff --git a/swh/loader/package/crates/loader.py b/swh/loader/package/crates/loader.py
--- a/swh/loader/package/crates/loader.py
+++ b/swh/loader/package/crates/loader.py
@@ -209,43 +209,46 @@
self,
storage: StorageInterface,
url: str,
- package_name: str,
- version: str,
- checksum: Optional[str] = None,
+ artifacts: List[Dict[str, Any]],
max_content_size: Optional[int] = None,
):
"""Constructor
Args:
- url : str
- Origin url (e.g.
- https://static.crates.io/crates/<name>/<name>-<version>.crate)
+ url:
+ Origin url, (e.g. https://crates.io/api/v1/crates/<package_name>)
- package_name : str
- Crate package name
+ artifacts:
+ A list of dict listing all existing released versions for a
+ package (Usually set with crates lister `extra_loader_arguments`).
+ Each line is a dict that should have an `url`
+ (where to download package specific version) and a `version` entry.
- version : str
- Crate package version
- checksum : str, optional
- Checksum for the package file to download
- """
+ Example::
+
+ [
+ {
+ "version": <version>,
+ "url": "https://static.crates.io/crates/<package_name>/<package_name>-<version>.crate",
+ }
+ ]
+ """ # noqa
super().__init__(storage=storage, url=url, max_content_size=max_content_size)
- self.name = package_name
- self.provider_url = f"https://crates.io/api/v1/crates/{self.name}"
- # Check consistency of name, version, url
- filename = urlparse(url).path.split("/")[-1]
- assert f"{self.name}-{version}.crate" == filename
+ self.url = url
+ self.artifacts: Dict[str, Dict] = {
+ artifact["version"]: artifact for artifact in artifacts
+ }
@cached_method
def _raw_info(self) -> bytes:
- """Get crate metadata (fetched from http api endpoint set as self.provider_url)
+ """Get crate metadata (fetched from http api endpoint set as self.url)
Returns:
Content response as bytes. Content response is a json document.
"""
- return api_info(self.provider_url)
+ return api_info(self.url)
@cached_method
def info(self) -> Dict:
@@ -263,7 +266,7 @@
["0.1.1", "0.10.2"]
"""
- versions = [version["num"] for version in self.info()["versions"]]
+ versions = list(self.artifacts.keys())
versions.sort(key=StrictVersion)
return versions
@@ -277,7 +280,7 @@
"0.1.2"
"""
- return self.info()["crate"]["newest_version"]
+ return self.get_versions()[-1]
def get_package_info(self, version: str) -> Iterator[Tuple[str, CratesPackageInfo]]:
"""Get release name and package information from version
@@ -288,13 +291,14 @@
Returns:
Iterator of tuple (release_name, p_info)
"""
- filename = f"{self.name}-{version}.crate"
- url = f"https://static.crates.io/crates/{self.name}/{self.name}-{version}.crate"
+ artifact = self.artifacts[version]
+ filename = artifact["filename"]
+ package_name = urlparse(self.url).path.split("/")[-1]
+ url = artifact["url"]
# Get extrinsic metadata from http api
-
- # Raw crate info
e_metadata = ExtrinsicPackageMetadata(**self.info()) # type: ignore[misc]
+
# Extract crate info for current version (One .crate file for a given version)
(crate_version,) = [
crate for crate in e_metadata["versions"] if crate["num"] == version
@@ -304,7 +308,7 @@
)
p_info = CratesPackageInfo(
- name=self.name,
+ name=package_name,
filename=filename,
url=url,
version=version,
@@ -324,7 +328,6 @@
# Get only corresponding key of IntrinsicPackageMetadata
i_metadata_keys = [k for k in IntrinsicPackageMetadata.__annotations__.keys()]
# We use data only from "package" entry
-
i_metadata = {
k: v for k, v in i_metadata_raw["package"].items() if k in i_metadata_keys
}
diff --git a/swh/loader/package/crates/tasks.py b/swh/loader/package/crates/tasks.py
--- a/swh/loader/package/crates/tasks.py
+++ b/swh/loader/package/crates/tasks.py
@@ -9,8 +9,6 @@
@shared_task(name=__name__ + ".LoadCrates")
-def load_crates(*, url=None, package_name: str, version: str, checksum=None):
+def load_crates(*, url=None, artifacts: list):
"""Load Rust crate package"""
- return CratesLoader.from_configfile(
- url=url, package_name=package_name, version=version, checksum=checksum
- ).load()
+ return CratesLoader.from_configfile(url=url, artifacts=artifacts).load()
diff --git a/swh/loader/package/crates/tests/test_crates.py b/swh/loader/package/crates/tests/test_crates.py
--- a/swh/loader/package/crates/tests/test_crates.py
+++ b/swh/loader/package/crates/tests/test_crates.py
@@ -2,6 +2,8 @@
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import pytest
+
from swh.loader.package.crates.loader import CratesLoader
from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats
from swh.model.hashutil import hash_to_bytes
@@ -17,16 +19,86 @@
CRATES_EXTRA = [
{
- "name": "hg-core",
- "version": "0.0.1",
- "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate",
- "checksum": "7fe168efadebadb9da6a329fdc027036e233b662285730cad27220e11e53c384",
+ "url": "https://crates.io/api/v1/crates/hg-core",
+ "artifacts": [
+ {
+ "checksums": {
+ "sha256": "48a45b46c2a8c38348adb1205b13c3c5eb0174e0c0fec52cc88e9fb1de14c54d", # noqa: B950
+ },
+ "filename": "hg-core-0.0.1.crate",
+ "url": "https://static.crates.io/crates/hg-core/hg-core-0.0.1.crate",
+ "version": "0.0.1",
+ },
+ ],
},
{
- "name": "micro-timer",
- "version": "0.4.0",
- "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate",
- "checksum": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405",
+ "url": "https://crates.io/api/v1/crates/micro-timer",
+ "artifacts": [
+ {
+ "checksums": {
+ "sha256": "69ad8fd116f8af0298ae4e83e587b1600af12709022471e25581c3aeb1da77ce", # noqa: B950
+ },
+ "filename": "micro-timer-0.1.0.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.0.crate",
+ "version": "0.1.0",
+ },
+ {
+ "checksums": {
+ "sha256": "7b3f65fe0e109daad8d47e1938c9b5f9353efacd86bbe7ff013f84ae7ca758bf", # noqa: B950
+ },
+ "filename": "micro-timer-0.1.1.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.1.crate",
+ "version": "0.1.1",
+ },
+ {
+ "checksums": {
+ "sha256": "16439fea388f712c1df7737ceb8f784d407844624b4796faf1e1bf8bbaa97445", # noqa: B950
+ },
+ "filename": "micro-timer-0.1.2.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.1.2.crate",
+ "version": "0.1.2",
+ },
+ {
+ "checksums": {
+ "sha256": "336b4c0f071d16674747faa4643d742cc096fec2bf8cf01bb1a98d984bedcaf1", # noqa: B950
+ },
+ "filename": "micro-timer-0.2.0.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.0.crate",
+ "version": "0.2.0",
+ },
+ {
+ "checksums": {
+ "sha256": "987429cd6162a80ed5ff44fc790f5090b1c6d617ac73a2e272965ed91201d79b", # noqa: B950
+ },
+ "filename": "micro-timer-0.2.1.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.2.1.crate",
+ "version": "0.2.1",
+ },
+ {
+ "checksums": {
+ "sha256": "25b31d6cb9112984323d05d7a353f272ae5d7a307074f9ab9b25c00121b8c947", # noqa: B950
+ },
+ "filename": "micro-timer-0.3.0.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.0.crate",
+ "version": "0.3.0",
+ },
+ {
+ "checksums": {
+ "sha256": "2620153e1d903d26b72b89f0e9c48d8c4756cba941c185461dddc234980c298c", # noqa: B950
+ },
+ "filename": "micro-timer-0.3.1.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.3.1.crate",
+ "version": "0.3.1",
+ },
+ {
+ "checksums": {
+ "sha256": "5de32cb59a062672560d6f0842c4aa7714727457b9fe2daf8987d995a176a405", # noqa: B950
+ },
+ "filename": "micro-timer-0.4.0.crate",
+ "url": "https://static.crates.io/crates/micro-timer/micro-timer-0.4.0.crate",
+ "version": "0.4.0",
+ },
+ ],
},
]
@@ -35,8 +107,7 @@
loader = CratesLoader(
swh_storage,
url=CRATES_EXTRA[1]["url"],
- package_name=CRATES_EXTRA[1]["name"],
- version=CRATES_EXTRA[1]["version"],
+ artifacts=CRATES_EXTRA[1]["artifacts"],
)
assert loader.get_versions() == [
"0.1.0",
@@ -54,34 +125,37 @@
loader = CratesLoader(
swh_storage,
url=CRATES_EXTRA[1]["url"],
- package_name=CRATES_EXTRA[1]["name"],
- version=CRATES_EXTRA[1]["version"],
+ artifacts=CRATES_EXTRA[1]["artifacts"],
)
assert loader.get_default_version() == "0.4.0"
-def test_crate_origin_not_found(swh_storage, requests_mock_datadir):
- url = "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate"
+def test_crate_invalid_origin_archive_not_found(swh_storage, requests_mock_datadir):
+ url = "https://nowhere-to-run/nowhere-to-hide"
loader = CratesLoader(
swh_storage,
url,
- package_name="nowhere-to-hide",
- version="0.0.1",
+ artifacts=[
+ {
+ "filename": "nowhere-to-hide-0.0.1.crate",
+ "url": "https://nowhere-to-run/nowhere-to-hide-0.0.1.crate",
+ "version": "0.0.1",
+ },
+ ],
)
- assert loader.load() == {"status": "failed"}
-
- assert_last_visit_matches(
- swh_storage, url, status="not_found", type="crates", snapshot=None
- )
+ with pytest.raises(Exception):
+ assert loader.load() == {"status": "failed"}
+ assert_last_visit_matches(
+ swh_storage, url, status="not_found", type="crates", snapshot=None
+ )
def test_crates_loader_load_one_version(datadir, requests_mock_datadir, swh_storage):
loader = CratesLoader(
swh_storage,
url=CRATES_EXTRA[0]["url"],
- package_name=CRATES_EXTRA[0]["name"],
- version=CRATES_EXTRA[0]["version"],
+ artifacts=CRATES_EXTRA[0]["artifacts"],
)
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
@@ -137,11 +211,10 @@
url = CRATES_EXTRA[1]["url"]
loader = CratesLoader(
swh_storage,
- url=url,
- package_name=CRATES_EXTRA[1]["name"],
- version=CRATES_EXTRA[1]["version"],
- checksum=CRATES_EXTRA[1]["checksum"],
+ url=CRATES_EXTRA[1]["url"],
+ artifacts=CRATES_EXTRA[1]["artifacts"],
)
+
actual_load_status = loader.load()
assert actual_load_status["status"] == "eventful"
assert actual_load_status["snapshot_id"] is not None
diff --git a/swh/loader/package/crates/tests/test_tasks.py b/swh/loader/package/crates/tests/test_tasks.py
--- a/swh/loader/package/crates/tests/test_tasks.py
+++ b/swh/loader/package/crates/tests/test_tasks.py
@@ -13,9 +13,8 @@
res = swh_scheduler_celery_app.send_task(
"swh.loader.package.crates.tasks.LoadCrates",
kwargs=dict(
- url="some-url/crates/some-package/some-package-0.0.1.crate",
- package_name="some-package",
- version="0.0.1",
+ url="some-url/api/v1/crates/some-package",
+ artifacts=[{"version": "0.0.1", "url": "some-package-0.0.1.crate"}],
),
)
assert res

File Metadata

Mime Type
text/plain
Expires
Thu, Jul 3, 2:00 PM (3 d, 3 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3216501

Event Timeline

About · Developer information · Legal