Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/loader.py
| Show All 22 Lines | from typing import ( | ||||
| Optional, | Optional, | ||||
| Sequence, | Sequence, | ||||
| Set, | Set, | ||||
| Tuple, | Tuple, | ||||
| TypeVar, | TypeVar, | ||||
| ) | ) | ||||
| import attr | import attr | ||||
| from requests.exceptions import ContentDecodingError | |||||
| import sentry_sdk | import sentry_sdk | ||||
| from swh.core.tarball import uncompress | from swh.core.tarball import uncompress | ||||
| from swh.loader.core.loader import BaseLoader | from swh.loader.core.loader import BaseLoader | ||||
| from swh.loader.exception import NotFound | from swh.loader.exception import NotFound | ||||
| from swh.loader.package.utils import download | from swh.loader.package.utils import download | ||||
| from swh.model import from_disk | from swh.model import from_disk | ||||
| from swh.model.collections import ImmutableDict | from swh.model.collections import ImmutableDict | ||||
| ▲ Show 20 Lines • Show All 317 Lines • ▼ Show 20 Lines | ) -> List[Tuple[str, Mapping]]: | ||||
| artifacts_package_info: Information on the package artifacts to | artifacts_package_info: Information on the package artifacts to | ||||
| download (url, filename, etc...) | download (url, filename, etc...) | ||||
| tmpdir: Location to retrieve such artifacts | tmpdir: Location to retrieve such artifacts | ||||
| Returns: | Returns: | ||||
| List of (path, computed hashes) | List of (path, computed hashes) | ||||
| """ | """ | ||||
| try: | |||||
| return [download(p_info.url, dest=tmpdir, filename=p_info.filename)] | return [download(p_info.url, dest=tmpdir, filename=p_info.filename)] | ||||
| except ContentDecodingError: | |||||
| # package might be erroneously marked as gzip compressed while is is not, | |||||
| # try to download its raw bytes again without attempting to uncompress | |||||
| # the input stream | |||||
| return [ | |||||
| download( | |||||
| p_info.url, | |||||
| dest=tmpdir, | |||||
| filename=p_info.filename, | |||||
| extra_request_headers={"Accept-Encoding": "identity"}, | |||||
| ) | |||||
| ] | |||||
| def uncompress( | def uncompress( | ||||
| self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | self, dl_artifacts: List[Tuple[str, Mapping[str, Any]]], dest: str | ||||
| ) -> str: | ) -> str: | ||||
| """Uncompress the artifact(s) in the destination folder dest. | """Uncompress the artifact(s) in the destination folder dest. | ||||
| Optionally, this could need to use the p_info dict for some more | Optionally, this could need to use the p_info dict for some more | ||||
| information (debian). | information (debian). | ||||
| ▲ Show 20 Lines • Show All 605 Lines • Show Last 20 Lines | |||||