Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
Show All 22 Lines | from typing import ( | ||||
Optional, | Optional, | ||||
Sequence, | Sequence, | ||||
Set, | Set, | ||||
Tuple, | Tuple, | ||||
TypeVar, | TypeVar, | ||||
) | ) | ||||
import attr | import attr | ||||
from requests.exceptions import ContentDecodingError | |||||
import sentry_sdk | import sentry_sdk | ||||
from swh.core.tarball import uncompress | from swh.core.tarball import uncompress | ||||
from swh.loader.core.loader import BaseLoader | from swh.loader.core.loader import BaseLoader | ||||
from swh.loader.exception import NotFound | from swh.loader.exception import NotFound | ||||
from swh.loader.package.utils import download | from swh.loader.package.utils import download | ||||
from swh.model import from_disk | from swh.model import from_disk | ||||
from swh.model.collections import ImmutableDict | from swh.model.collections import ImmutableDict | ||||
▲ Show 20 Lines • Show All 317 Lines • ▼ Show 20 Lines | ) -> List[Tuple[str, Mapping]]: | ||||
artifacts_package_info: Information on the package artifacts to | artifacts_package_info: Information on the package artifacts to | ||||
download (url, filename, etc...) | download (url, filename, etc...) | ||||
tmpdir: Location to retrieve such artifacts | tmpdir: Location to retrieve such artifacts | ||||
Returns: | Returns: | ||||
List of (path, computed hashes) | List of (path, computed hashes) | ||||
""" | """ | ||||
try: | |||||
return [download(p_info.url, dest=tmpdir, filename=p_info.filename)] | return [download(p_info.url, dest=tmpdir, filename=p_info.filename)] | ||||
except ContentDecodingError: | |||||
# package might be erroneously marked as gzip compressed while is is not, | |||||
# try to download its raw bytes again without attempting to uncompress | |||||
# the input stream | |||||
return [ | |||||
download( | |||||
p_info.url, | |||||
dest=tmpdir, | |||||
filename=p_info.filename, | |||||
extra_request_headers={"Accept-Encoding": "identity"}, | |||||
) | |||||
] | |||||
def uncompress( | def uncompress( | ||||
self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | ||||
) -> str: | ) -> str: | ||||
"""Uncompress the artifact(s) in the destination folder dest. | """Uncompress the artifact(s) in the destination folder dest. | ||||
Optionally, this could need to use the p_info dict for some more | Optionally, this could need to use the p_info dict for some more | ||||
information (debian). | information (debian). | ||||
▲ Show 20 Lines • Show All 605 Lines • Show Last 20 Lines |