Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/crates/loader.py
# Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from distutils.version import StrictVersion | from distutils.version import StrictVersion | ||||
import json | import json | ||||
from pathlib import Path | from pathlib import Path | ||||
from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple | from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple | ||||
from urllib.parse import urlparse | from urllib.parse import urlparse | ||||
import attr | import attr | ||||
import toml | import toml | ||||
from typing_extensions import TypedDict | from typing_extensions import TypedDict | ||||
from swh.loader.package.loader import BasePackageInfo, PackageLoader | from swh.loader.package.loader import BasePackageInfo, PackageLoader | ||||
from swh.loader.package.utils import api_info, cached_method, release_name | from swh.loader.package.utils import cached_method, get_url_body, release_name | ||||
from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone | from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
class ExtrinsicPackageMetadata(TypedDict): | class ExtrinsicPackageMetadata(TypedDict): | ||||
"""Data structure for package extrinsic metadata pulled from http api endpoint. | """Data structure for package extrinsic metadata pulled from http api endpoint. | ||||
We set only the keys we need according to what is available when querying | We set only the keys we need according to what is available when querying | ||||
▲ Show 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | class CratesLoader(PackageLoader[CratesPackageInfo]): | ||||
@cached_method | @cached_method | ||||
def _raw_info(self) -> bytes: | def _raw_info(self) -> bytes: | ||||
"""Get crate metadata (fetched from http api endpoint set as self.url) | """Get crate metadata (fetched from http api endpoint set as self.url) | ||||
Returns: | Returns: | ||||
Content response as bytes. Content response is a json document. | Content response as bytes. Content response is a json document. | ||||
""" | """ | ||||
return api_info(self.url) | return get_url_body(self.url) | ||||
@cached_method | @cached_method | ||||
def info(self) -> Dict: | def info(self) -> Dict: | ||||
"""Parse http api json response and return the crate metadata information | """Parse http api json response and return the crate metadata information | ||||
as a Dict.""" | as a Dict.""" | ||||
return json.loads(self._raw_info()) | return json.loads(self._raw_info()) | ||||
def get_versions(self) -> Sequence[str]: | def get_versions(self) -> Sequence[str]: | ||||
▲ Show 20 Lines • Show All 95 Lines • Show Last 20 Lines |