Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show First 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | class BasePackageInfo: | ||||
| directory_extrinsic_metadata = attr.ib( | directory_extrinsic_metadata = attr.ib( | ||||
| type=List[RawExtrinsicMetadataCore], | type=List[RawExtrinsicMetadataCore], | ||||
| default=[], | default=[], | ||||
| kw_only=True, | kw_only=True, | ||||
| ) | ) | ||||
| """:term:`extrinsic metadata` collected by the loader, that will be attached to the | """:term:`extrinsic metadata` collected by the loader, that will be attached to the | ||||
| loaded directory and added to the Metadata storage.""" | loaded directory and added to the Metadata storage.""" | ||||
| checksums = attr.ib(type=Dict[str, str], default={}, kw_only=True) | |||||
| """Dictionary holding package tarball checksums for integrity check after | |||||
| download, keys are hash algorithm names and values are checksums in | |||||
| hexadecimal format. The supported algorithms are defined in the | |||||
| :data:`swh.model.hashutil.ALGORITHMS` set.""" | |||||
| # TODO: add support for metadata for releases and contents | # TODO: add support for metadata for releases and contents | ||||
| def extid(self) -> Optional[PartialExtID]: | def extid(self) -> Optional[PartialExtID]: | ||||
| """Returns a unique intrinsic identifier of this package info, | """Returns a unique intrinsic identifier of this package info, | ||||
| or None if this package info is not 'deduplicatable' (meaning that | or None if this package info is not 'deduplicatable' (meaning that | ||||
| we will always load it, instead of checking the ExtID storage | we will always load it, instead of checking the ExtID storage | ||||
| to see if we already did)""" | to see if we already did)""" | ||||
| if self.MANIFEST_FORMAT is None: | if self.MANIFEST_FORMAT is None: | ||||
| ▲ Show 20 Lines • Show All 266 Lines • ▼ Show 20 Lines | ) -> List[Tuple[str, Mapping]]: | ||||
| download (url, filename, etc...) | download (url, filename, etc...) | ||||
| tmpdir: Location to retrieve such artifacts | tmpdir: Location to retrieve such artifacts | ||||
| Returns: | Returns: | ||||
| List of (path, computed hashes) | List of (path, computed hashes) | ||||
| """ | """ | ||||
| try: | try: | ||||
| return [download(p_info.url, dest=tmpdir, filename=p_info.filename)] | return [ | ||||
| download( | |||||
| p_info.url, | |||||
| dest=tmpdir, | |||||
| filename=p_info.filename, | |||||
| hashes=p_info.checksums, | |||||
| ) | |||||
| ] | |||||
| except ContentDecodingError: | except ContentDecodingError: | ||||
| # package might be erroneously marked as gzip compressed while is is not, | # package might be erroneously marked as gzip compressed while is is not, | ||||
| # try to download its raw bytes again without attempting to uncompress | # try to download its raw bytes again without attempting to uncompress | ||||
| # the input stream | # the input stream | ||||
| return [ | return [ | ||||
| download( | download( | ||||
| p_info.url, | p_info.url, | ||||
| dest=tmpdir, | dest=tmpdir, | ||||
| filename=p_info.filename, | filename=p_info.filename, | ||||
| hashes=p_info.checksums, | |||||
| extra_request_headers={"Accept-Encoding": "identity"}, | extra_request_headers={"Accept-Encoding": "identity"}, | ||||
| ) | ) | ||||
| ] | ] | ||||
| def uncompress( | def uncompress( | ||||
| self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | ||||
| ) -> str: | ) -> str: | ||||
| """Uncompress the artifact(s) in the destination folder dest. | """Uncompress the artifact(s) in the destination folder dest. | ||||
| ▲ Show 20 Lines • Show All 693 Lines • Show Last 20 Lines | |||||