Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/golang/loader.py
| # Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import re | |||||
| from typing import Iterator, Optional, Sequence, Tuple | from typing import Iterator, Optional, Sequence, Tuple | ||||
| import attr | import attr | ||||
| from swh.loader.package.loader import BasePackageInfo, PackageLoader | from swh.loader.package.loader import BasePackageInfo, PackageLoader | ||||
| from swh.loader.package.utils import EMPTY_AUTHOR, api_info, release_name | from swh.loader.package.utils import EMPTY_AUTHOR, api_info, release_name | ||||
| from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| def _uppercase_encode(url: str) -> str: | |||||
| return re.sub("([A-Z]{1})", r"!\1", url).lower() | |||||
| @attr.s | @attr.s | ||||
| class GolangPackageInfo(BasePackageInfo): | class GolangPackageInfo(BasePackageInfo): | ||||
| name = attr.ib(type=str) | name = attr.ib(type=str) | ||||
| timestamp = attr.ib(type=Optional[TimestampWithTimezone]) | timestamp = attr.ib(type=Optional[TimestampWithTimezone]) | ||||
| class GolangLoader(PackageLoader[GolangPackageInfo]): | class GolangLoader(PackageLoader[GolangPackageInfo]): | ||||
| """Load Golang module zip file into SWH archive.""" | """Load Golang module zip file into SWH archive.""" | ||||
| Show All 13 Lines | ): | ||||
| # The lister saves human-usable URLs, so we translate them to proxy URLs | # The lister saves human-usable URLs, so we translate them to proxy URLs | ||||
| # for use in the loader. | # for use in the loader. | ||||
| # This URL format is detailed in https://go.dev/ref/mod#goproxy-protocol | # This URL format is detailed in https://go.dev/ref/mod#goproxy-protocol | ||||
| assert url.startswith( | assert url.startswith( | ||||
| self.GOLANG_PKG_DEV_URL | self.GOLANG_PKG_DEV_URL | ||||
| ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ||||
| self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | ||||
| self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | ||||
| self.url = _uppercase_encode(self.url) | |||||
| def get_versions(self) -> Sequence[str]: | def get_versions(self) -> Sequence[str]: | ||||
| return api_info(f"{self.url}/@v/list").decode().splitlines() | return api_info(f"{self.url}/@v/list").decode().splitlines() | ||||
| def get_default_version(self) -> str: | def get_default_version(self) -> str: | ||||
| latest = api_info(f"{self.url}/@latest") | latest = api_info(f"{self.url}/@latest") | ||||
| return json.loads(latest)["Version"] | return json.loads(latest)["Version"] | ||||
| def _raw_info(self, version: str) -> dict: | def _raw_info(self, version: str) -> dict: | ||||
| url = f"{self.url}/@v/{version}.info" | url = f"{self.url}/@v/{_uppercase_encode(version)}.info" | ||||
| return json.loads(api_info(url)) | return json.loads(api_info(url)) | ||||
| def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | ||||
| # Encode the name because creating nested folders can become problematic | # Encode the name because creating nested folders can become problematic | ||||
| encoded_name = self.name.replace("/", "__") | encoded_name = self.name.replace("/", "__") | ||||
| filename = f"{encoded_name}-{version}.zip" | filename = f"{encoded_name}-{version}.zip" | ||||
| timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | ||||
| p_info = GolangPackageInfo( | p_info = GolangPackageInfo( | ||||
| Show All 25 Lines | |||||