Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/golang/loader.py
- This file was added.
| # Copyright (C) 2022 The Software Heritage developers | |||||
| # See the AUTHORS file at the top-level directory of this distribution | |||||
| # License: GNU General Public License version 3, or any later version | |||||
| # See top-level LICENSE file for more information | |||||
| import json | |||||
| import logging | |||||
| from typing import Iterator, Optional, Sequence, Tuple | |||||
| import attr | |||||
| from swh.loader.package.loader import BasePackageInfo, PackageLoader | |||||
| from swh.loader.package.utils import EMPTY_AUTHOR, api_info, cached_method, release_name | |||||
| from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | |||||
| from swh.storage.interface import StorageInterface | |||||
| logger = logging.getLogger(__name__) | |||||
| @attr.s | |||||
| class GolangPackageInfo(BasePackageInfo): | |||||
| name = attr.ib(type=str) | |||||
| timestamp = attr.ib(type=Optional[TimestampWithTimezone]) | |||||
| class GolangLoader(PackageLoader[GolangPackageInfo]): | |||||
| """Load Golang module zip file into SWH archive.""" | |||||
| visit_type = "golang" | |||||
| GOLANG_PROXY_URL = "https://proxy.golang.org" | |||||
| def __init__( | |||||
| self, | |||||
| storage: StorageInterface, | |||||
| url: str, | |||||
| max_content_size: Optional[int] = None, | |||||
| ): | |||||
| super().__init__(storage, url, max_content_size) | |||||
| # URL format is detailed in https://go.dev/ref/mod#goproxy-protocol | |||||
| # The lister gives us the zip URL without the proxy base, but we may | |||||
| # want to support the full URL as well | |||||
| url = url.replace(self.GOLANG_PROXY_URL, "") | |||||
| self.name, self.version = url.replace(".zip", "").split("/@v/", 1) | |||||
| def get_versions(self) -> Sequence[str]: | |||||
| return ( | |||||
| api_info(f"{self.GOLANG_PROXY_URL}/{self.name}/@v/list") | |||||
| .decode() | |||||
| .splitlines() | |||||
| ) | |||||
| def get_default_version(self) -> str: | |||||
| latest = api_info(f"{self.GOLANG_PROXY_URL}/{self.name}/@latest") | |||||
| return json.loads(latest)["Version"] | |||||
| @cached_method | |||||
| def _raw_info(self) -> dict: | |||||
| url = f"{self.GOLANG_PROXY_URL}/{self.name}/@v/{self.version}.info" | |||||
| return json.loads(api_info(url)) | |||||
| def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | |||||
| # Encode the url because creating nested folders can become problematic | |||||
| filename = self.url.replace("/", "__") | |||||
| timestamp = TimestampWithTimezone.from_iso8601(self._raw_info()["Time"]) | |||||
| p_info = GolangPackageInfo( | |||||
| url=f"{self.GOLANG_PROXY_URL}/{self.url}", | |||||
| filename=filename, | |||||
| version=self.version, | |||||
| timestamp=timestamp, | |||||
| name=self.name, | |||||
| ) | |||||
| yield release_name(version, filename), p_info | |||||
| def build_release( | |||||
| self, p_info: GolangPackageInfo, uncompressed_path: str, directory: Sha1Git | |||||
| ) -> Optional[Release]: | |||||
| msg = ( | |||||
| f"Synthetic release for Golang source package {p_info.name} " | |||||
| f"version {p_info.version}\n" | |||||
| ) | |||||
| return Release( | |||||
| name=p_info.version.encode(), | |||||
| message=msg.encode(), | |||||
| date=p_info.timestamp, | |||||
| author=EMPTY_AUTHOR, # Go modules offer very little metadata | |||||
| target_type=ObjectType.DIRECTORY, | |||||
| target=directory, | |||||
| synthetic=True, | |||||
| ) |