Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/golang/loader.py
| # Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import re | import re | ||||
| from typing import Iterator, Optional, Sequence, Tuple | from typing import Iterator, Optional, Sequence, Tuple | ||||
| import attr | import attr | ||||
| from swh.loader.package.loader import BasePackageInfo, PackageLoader | from swh.loader.package.loader import BasePackageInfo, PackageLoader | ||||
| from swh.loader.package.utils import EMPTY_AUTHOR, api_info, cached_method, release_name | from swh.loader.package.utils import ( | ||||
| EMPTY_AUTHOR, | |||||
| get_url_body, | |||||
| release_name, | |||||
| cached_method, | |||||
| ) | |||||
| from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| def _uppercase_encode(url: str) -> str: | def _uppercase_encode(url: str) -> str: | ||||
| return re.sub("([A-Z]{1})", r"!\1", url).lower() | return re.sub("([A-Z]{1})", r"!\1", url).lower() | ||||
| Show All 26 Lines | ): | ||||
| assert url.startswith( | assert url.startswith( | ||||
| self.GOLANG_PKG_DEV_URL | self.GOLANG_PKG_DEV_URL | ||||
| ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ||||
| self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | ||||
| self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | ||||
| self.url = _uppercase_encode(self.url) | self.url = _uppercase_encode(self.url) | ||||
| def get_versions(self) -> Sequence[str]: | def get_versions(self) -> Sequence[str]: | ||||
| versions = api_info(f"{self.url}/@v/list").decode().splitlines() | versions = get_url_body(f"{self.url}/@v/list").decode().splitlines() | ||||
| # some go packages only have a development version not listed by the endpoint above, | # some go packages only have a development version not listed by the endpoint above, | ||||
| # so ensure to return it or it will be missed by the golang loader | # so ensure to return it or it will be missed by the golang loader | ||||
| default_version = self.get_default_version() | default_version = self.get_default_version() | ||||
| if default_version not in versions: | if default_version not in versions: | ||||
| versions.append(default_version) | versions.append(default_version) | ||||
| return versions | return versions | ||||
| @cached_method | @cached_method | ||||
| def get_default_version(self) -> str: | def get_default_version(self) -> str: | ||||
| latest = api_info(f"{self.url}/@latest") | latest = get_url_body(f"{self.url}/@latest") | ||||
| return json.loads(latest)["Version"] | return json.loads(latest)["Version"] | ||||
| def _raw_info(self, version: str) -> dict: | def _raw_info(self, version: str) -> dict: | ||||
| url = f"{self.url}/@v/{_uppercase_encode(version)}.info" | url = f"{self.url}/@v/{_uppercase_encode(version)}.info" | ||||
| return json.loads(api_info(url)) | return json.loads(get_url_body(url)) | ||||
| def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | ||||
| # Encode the name because creating nested folders can become problematic | # Encode the name because creating nested folders can become problematic | ||||
| encoded_name = self.name.replace("/", "__") | encoded_name = self.name.replace("/", "__") | ||||
| filename = f"{encoded_name}-{version}.zip" | filename = f"{encoded_name}-{version}.zip" | ||||
| timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | ||||
| p_info = GolangPackageInfo( | p_info = GolangPackageInfo( | ||||
| url=f"{self.url}/@v/{version}.zip", | url=f"{self.url}/@v/{version}.zip", | ||||
| Show All 24 Lines | |||||