Differential D5071 Diff 18127 swh/loader/package/loader.py

Changeset View

Standalone View

swh/loader/package/loader.py

Show All 21 Lines	from typing import (
Sequence,		Sequence,
Tuple,		Tuple,
TypeVar,		TypeVar,
)		)

import attr		import attr
import sentry_sdk		import sentry_sdk

from swh.core.config import load_from_envvar
from swh.core.tarball import uncompress		from swh.core.tarball import uncompress
from swh.loader.exception import NotFound		from swh.loader.exception import NotFound
from swh.loader.package.utils import download		from swh.loader.package.utils import download
		from swh.loader.pattern import Loader
from swh.model import from_disk		from swh.model import from_disk
from swh.model.collections import ImmutableDict		from swh.model.collections import ImmutableDict
from swh.model.hashutil import hash_to_hex		from swh.model.hashutil import hash_to_hex
from swh.model.identifiers import SWHID		from swh.model.identifiers import SWHID
from swh.model.model import (		from swh.model.model import (
MetadataAuthority,		MetadataAuthority,
MetadataAuthorityType,		MetadataAuthorityType,
MetadataFetcher,		MetadataFetcher,
MetadataTargetType,		MetadataTargetType,
Origin,		Origin,
OriginVisit,		OriginVisit,
OriginVisitStatus,		OriginVisitStatus,
RawExtrinsicMetadata,		RawExtrinsicMetadata,
Revision,		Revision,
Sha1Git,		Sha1Git,
Snapshot,		Snapshot,
TargetType,		TargetType,
)		)
from swh.storage import get_storage
from swh.storage.algos.snapshot import snapshot_get_latest		from swh.storage.algos.snapshot import snapshot_get_latest
from swh.storage.interface import StorageInterface		from swh.storage.interface import StorageInterface
from swh.storage.utils import now		from swh.storage.utils import now

logger = logging.getLogger(__name__)		logger = logging.getLogger(__name__)


SWH_METADATA_AUTHORITY = MetadataAuthority(		SWH_METADATA_AUTHORITY = MetadataAuthority(
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	class BasePackageInfo:

def artifact_identity(self):		def artifact_identity(self):
return [getattr(self, k) for k in self.ID_KEYS]		return [getattr(self, k) for k in self.ID_KEYS]


TPackageInfo = TypeVar("TPackageInfo", bound=BasePackageInfo)		TPackageInfo = TypeVar("TPackageInfo", bound=BasePackageInfo)


DEFAULT_CONFIG = {		class PackageLoader(Loader, Generic[TPackageInfo]):
"max_content_size": 100 * 1024 * 1024,
"create_authorities": True,
"create_fetchers": True,
}


class PackageLoader(Generic[TPackageInfo]):
# Origin visit type (str) set by the loader		# Origin visit type (str) set by the loader
visit_type = ""		visit_type = ""

def __init__(self, url):		def __init__(
		self,
		storage: StorageInterface,
		url: str,
		max_content_size: Optional[int] = None,
		create_authorities: bool = True,
		olasdUnsubmitted Not Done Inline Actions You're dropping the only use of these parameters in tests, so you might as well drop them altogether. olasd: You're dropping the only use of these parameters in tests, so you might as well drop them…
		create_fetchers: bool = True,
		):
"""Loader's constructor. This raises exception if the minimal required		"""Loader's constructor. This raises exception if the minimal required
configuration is missing (cf. fn:`check` method).		configuration is missing (cf. fn:`check` method).

Args:		Args:
url (str): Origin url to load data from		storage: Storage instance
		url: Origin url to load data from

"""		"""
# This expects to use the environment variable SWH_CONFIG_FILENAME		super().__init__(storage=storage, max_content_size=max_content_size)
self.config = load_from_envvar(DEFAULT_CONFIG)
self._check_configuration()
self.storage: StorageInterface = get_storage(**self.config["storage"])
self.url = url		self.url = url
self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc)		self.visit_date = datetime.datetime.now(tz=datetime.timezone.utc)
self.max_content_size = self.config["max_content_size"]		self.create_authorities = create_authorities
		self.create_fetchers = create_fetchers
def _check_configuration(self):
"""Checks the minimal configuration required is set for the loader.

If some required configuration is missing, exception detailing the
issue is raised.

"""
if "storage" not in self.config:
raise ValueError("Misconfiguration, at least the storage key should be set")

def get_versions(self) -> Sequence[str]:		def get_versions(self) -> Sequence[str]:
"""Return the list of all published package versions.		"""Return the list of all published package versions.

Raises:		Raises:
`class:swh.loader.exception.NotFound` error when failing to read the		`class:swh.loader.exception.NotFound` error when failing to read the
published package versions.		published package versions.

▲ Show 20 Lines • Show All 638 Lines • ▼ Show 20 Lines	) -> None:
self._create_fetchers(mo.fetcher for mo in metadata_objects)		self._create_fetchers(mo.fetcher for mo in metadata_objects)

self.storage.raw_extrinsic_metadata_add(metadata_objects)		self.storage.raw_extrinsic_metadata_add(metadata_objects)

def _create_authorities(self, authorities: Iterable[MetadataAuthority]) -> None:		def _create_authorities(self, authorities: Iterable[MetadataAuthority]) -> None:
deduplicated_authorities = {		deduplicated_authorities = {
(authority.type, authority.url): authority for authority in authorities		(authority.type, authority.url): authority for authority in authorities
}		}
if authorities:		if self.create_authorities and authorities:
self.storage.metadata_authority_add(list(deduplicated_authorities.values()))		self.storage.metadata_authority_add(list(deduplicated_authorities.values()))

def _create_fetchers(self, fetchers: Iterable[MetadataFetcher]) -> None:		def _create_fetchers(self, fetchers: Iterable[MetadataFetcher]) -> None:
deduplicated_fetchers = {		deduplicated_fetchers = {
(fetcher.name, fetcher.version): fetcher for fetcher in fetchers		(fetcher.name, fetcher.version): fetcher for fetcher in fetchers
}		}
if fetchers:		if self.create_fetchers and fetchers:
self.storage.metadata_fetcher_add(list(deduplicated_fetchers.values()))		self.storage.metadata_fetcher_add(list(deduplicated_fetchers.values()))