diff --git a/swh/loader/package/archive/loader.py b/swh/loader/package/archive/loader.py --- a/swh/loader/package/archive/loader.py +++ b/swh/loader/package/archive/loader.py @@ -72,8 +72,8 @@ url: str, artifacts: Sequence[Dict[str, Any]], extid_manifest_format: Optional[str] = None, - max_content_size: Optional[int] = None, snapshot_append: bool = False, + **kwargs: Any, ): f"""Loader constructor. @@ -101,7 +101,7 @@ the new snapshot created by the loader """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.artifacts = artifacts # assume order is enforced in the lister self.extid_manifest_format = ( None diff --git a/swh/loader/package/cran/loader.py b/swh/loader/package/cran/loader.py --- a/swh/loader/package/cran/loader.py +++ b/swh/loader/package/cran/loader.py @@ -51,11 +51,7 @@ visit_type = "cran" def __init__( - self, - storage: StorageInterface, - url: str, - artifacts: List[Dict], - max_content_size: Optional[int] = None, + self, storage: StorageInterface, url: str, artifacts: List[Dict], **kwargs: Any ): """Loader constructor. @@ -64,7 +60,7 @@ artifacts: List of associated artifact for the origin url """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) # explicit what we consider the artifact identity self.artifacts = artifacts diff --git a/swh/loader/package/debian/loader.py b/swh/loader/package/debian/loader.py --- a/swh/loader/package/debian/loader.py +++ b/swh/loader/package/debian/loader.py @@ -132,7 +132,7 @@ storage: StorageInterface, url: str, packages: Mapping[str, Any], - max_content_size: Optional[int] = None, + **kwargs: Any, ): """Debian Loader implementation. @@ -173,7 +173,7 @@ } """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.packages = packages def get_versions(self) -> Sequence[str]: diff --git a/swh/loader/package/deposit/loader.py b/swh/loader/package/deposit/loader.py --- a/swh/loader/package/deposit/loader.py +++ b/swh/loader/package/deposit/loader.py @@ -107,8 +107,8 @@ url: str, deposit_id: str, deposit_client: "ApiClient", - max_content_size: Optional[int] = None, default_filename: str = "archive.tar", + **kwargs: Any, ): """Constructor @@ -118,7 +118,7 @@ deposit_client: Deposit api client """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.deposit_id = deposit_id self.client = deposit_client diff --git a/swh/loader/package/loader.py b/swh/loader/package/loader.py --- a/swh/loader/package/loader.py +++ b/swh/loader/package/loader.py @@ -151,16 +151,7 @@ class PackageLoader(BaseLoader, Generic[TPackageInfo]): - # Origin visit type (str) set by the loader - visit_type = "" - visit_date: datetime.datetime - - def __init__( - self, - storage: StorageInterface, - url: str, - max_content_size: Optional[int] = None, - ): + def __init__(self, storage: StorageInterface, url: str, **kwargs: Any): """Loader's constructor. This raises exception if the minimal required configuration is missing (cf. fn:`check` method). @@ -169,10 +160,7 @@ url: Origin url to load data from """ - super().__init__( - storage=storage, origin_url=url, max_content_size=max_content_size - ) - self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc) + super().__init__(storage=storage, origin_url=url, **kwargs) def get_versions(self) -> Sequence[str]: """Return the list of all published package versions. diff --git a/swh/loader/package/maven/loader.py b/swh/loader/package/maven/loader.py --- a/swh/loader/package/maven/loader.py +++ b/swh/loader/package/maven/loader.py @@ -8,7 +8,7 @@ import logging from os import path import string -from typing import Iterator, List, Optional, Sequence, Tuple +from typing import Any, Iterator, List, Optional, Sequence, Tuple import attr import iso8601 @@ -110,7 +110,7 @@ storage: StorageInterface, url: str, artifacts: Sequence[ArtifactDict], - max_content_size: Optional[int] = None, + **kwargs: Any, ): """Loader constructor. @@ -123,7 +123,7 @@ artifacts: List of single artifact information """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.artifacts = artifacts # assume order is enforced in the lister self.version_artifact = { jar["version"]: jar for jar in artifacts if jar["version"] diff --git a/swh/loader/package/nixguix/loader.py b/swh/loader/package/nixguix/loader.py --- a/swh/loader/package/nixguix/loader.py +++ b/swh/loader/package/nixguix/loader.py @@ -75,9 +75,9 @@ storage: StorageInterface, url: str, unsupported_file_extensions: List[str] = [], - max_content_size: Optional[int] = None, + **kwargs: Any, ): - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.provider_url = url self.unsupported_file_extensions = unsupported_file_extensions diff --git a/swh/loader/package/npm/loader.py b/swh/loader/package/npm/loader.py --- a/swh/loader/package/npm/loader.py +++ b/swh/loader/package/npm/loader.py @@ -101,18 +101,13 @@ visit_type = "npm" - def __init__( - self, - storage: StorageInterface, - url: str, - max_content_size: Optional[int] = None, - ): + def __init__(self, storage: StorageInterface, url: str, **kwargs: Any): """Constructor Args str: origin url (e.g. https://www.npmjs.com/package/) """ - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.package_name = url.split("https://www.npmjs.com/package/")[1] safe_name = quote(self.package_name, safe="") self.provider_url = f"https://replicate.npmjs.com/{safe_name}/" diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py --- a/swh/loader/package/opam/loader.py +++ b/swh/loader/package/opam/loader.py @@ -6,7 +6,7 @@ import io import os from subprocess import PIPE, Popen, call -from typing import Iterator, List, Optional, Tuple +from typing import Any, Iterator, List, Optional, Tuple import attr @@ -91,10 +91,10 @@ opam_instance: str, opam_url: str, opam_package: str, - max_content_size: Optional[int] = None, initialize_opam_root: bool = False, + **kwargs: Any, ): - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + super().__init__(storage=storage, url=url, **kwargs) self.opam_root = opam_root self.opam_instance = opam_instance diff --git a/swh/loader/package/pypi/loader.py b/swh/loader/package/pypi/loader.py --- a/swh/loader/package/pypi/loader.py +++ b/swh/loader/package/pypi/loader.py @@ -77,13 +77,8 @@ visit_type = "pypi" - def __init__( - self, - storage: StorageInterface, - url: str, - max_content_size: Optional[int] = None, - ): - super().__init__(storage=storage, url=url, max_content_size=max_content_size) + def __init__(self, storage: StorageInterface, url: str, **kwargs): + super().__init__(storage=storage, url=url, **kwargs) self.provider_url = pypi_api_url(self.origin.url) @cached_method diff --git a/swh/loader/package/tests/test_loader.py b/swh/loader/package/tests/test_loader.py --- a/swh/loader/package/tests/test_loader.py +++ b/swh/loader/package/tests/test_loader.py @@ -54,6 +54,8 @@ class StubPackageLoader(PackageLoader[StubPackageInfo]): + visit_type = "stub" + def get_versions(self): return ["v1.0", "v2.0", "v3.0", "v4.0"] diff --git a/swh/loader/package/tests/test_loader_metadata.py b/swh/loader/package/tests/test_loader_metadata.py --- a/swh/loader/package/tests/test_loader_metadata.py +++ b/swh/loader/package/tests/test_loader_metadata.py @@ -88,6 +88,8 @@ class MetadataTestLoader(PackageLoader[BasePackageInfo]): + visit_type = "metadata-test" + def get_versions(self) -> Sequence[str]: return ["v1.0.0"]