Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
Show All 21 Lines | from typing import ( | ||||
Sequence, | Sequence, | ||||
Tuple, | Tuple, | ||||
TypeVar, | TypeVar, | ||||
) | ) | ||||
import attr | import attr | ||||
import sentry_sdk | import sentry_sdk | ||||
from swh.core.config import load_from_envvar | |||||
from swh.core.tarball import uncompress | from swh.core.tarball import uncompress | ||||
from swh.loader.core.loader import Loader | |||||
from swh.loader.exception import NotFound | from swh.loader.exception import NotFound | ||||
from swh.loader.package.utils import download | from swh.loader.package.utils import download | ||||
from swh.model import from_disk | from swh.model import from_disk | ||||
from swh.model.collections import ImmutableDict | from swh.model.collections import ImmutableDict | ||||
from swh.model.hashutil import hash_to_hex | from swh.model.hashutil import hash_to_hex | ||||
from swh.model.identifiers import SWHID | from swh.model.identifiers import SWHID | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
MetadataAuthority, | MetadataAuthority, | ||||
MetadataAuthorityType, | MetadataAuthorityType, | ||||
MetadataFetcher, | MetadataFetcher, | ||||
MetadataTargetType, | MetadataTargetType, | ||||
Origin, | Origin, | ||||
OriginVisit, | OriginVisit, | ||||
OriginVisitStatus, | OriginVisitStatus, | ||||
RawExtrinsicMetadata, | RawExtrinsicMetadata, | ||||
Revision, | Revision, | ||||
Sha1Git, | Sha1Git, | ||||
Snapshot, | Snapshot, | ||||
TargetType, | TargetType, | ||||
) | ) | ||||
from swh.storage import get_storage | |||||
from swh.storage.algos.snapshot import snapshot_get_latest | from swh.storage.algos.snapshot import snapshot_get_latest | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
from swh.storage.utils import now | from swh.storage.utils import now | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
SWH_METADATA_AUTHORITY = MetadataAuthority( | SWH_METADATA_AUTHORITY = MetadataAuthority( | ||||
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | class BasePackageInfo: | ||||
def artifact_identity(self): | def artifact_identity(self): | ||||
return [getattr(self, k) for k in self.ID_KEYS] | return [getattr(self, k) for k in self.ID_KEYS] | ||||
TPackageInfo = TypeVar("TPackageInfo", bound=BasePackageInfo) | TPackageInfo = TypeVar("TPackageInfo", bound=BasePackageInfo) | ||||
DEFAULT_CONFIG = { | class PackageLoader(Loader, Generic[TPackageInfo]): | ||||
"max_content_size": 100 * 1024 * 1024, | |||||
"create_authorities": True, | |||||
"create_fetchers": True, | |||||
} | |||||
class PackageLoader(Generic[TPackageInfo]): | |||||
# Origin visit type (str) set by the loader | # Origin visit type (str) set by the loader | ||||
visit_type = "" | visit_type = "" | ||||
def __init__(self, url): | def __init__( | ||||
self, | |||||
storage: StorageInterface, | |||||
url: str, | |||||
max_content_size: Optional[int] = None, | |||||
): | |||||
olasd: You're dropping the only use of these parameters in tests, so you might as well drop them… | |||||
"""Loader's constructor. This raises exception if the minimal required | """Loader's constructor. This raises exception if the minimal required | ||||
configuration is missing (cf. fn:`check` method). | configuration is missing (cf. fn:`check` method). | ||||
Args: | Args: | ||||
url (str): Origin url to load data from | storage: Storage instance | ||||
url: Origin url to load data from | |||||
""" | """ | ||||
# This expects to use the environment variable SWH_CONFIG_FILENAME | super().__init__(storage=storage, max_content_size=max_content_size) | ||||
self.config = load_from_envvar(DEFAULT_CONFIG) | |||||
self._check_configuration() | |||||
self.storage: StorageInterface = get_storage(**self.config["storage"]) | |||||
self.url = url | self.url = url | ||||
self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
self.max_content_size = self.config["max_content_size"] | |||||
def _check_configuration(self): | |||||
"""Checks the minimal configuration required is set for the loader. | |||||
If some required configuration is missing, exception detailing the | |||||
issue is raised. | |||||
""" | |||||
if "storage" not in self.config: | |||||
raise ValueError("Misconfiguration, at least the storage key should be set") | |||||
def get_versions(self) -> Sequence[str]: | def get_versions(self) -> Sequence[str]: | ||||
"""Return the list of all published package versions. | """Return the list of all published package versions. | ||||
Raises: | Raises: | ||||
`class:swh.loader.exception.NotFound` error when failing to read the | `class:swh.loader.exception.NotFound` error when failing to read the | ||||
published package versions. | published package versions. | ||||
▲ Show 20 Lines • Show All 655 Lines • Show Last 20 Lines |
You're dropping the only use of these parameters in tests, so you might as well drop them altogether.