Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/golang/loader.py
# Copyright (C) 2022 The Software Heritage developers | # Copyright (C) 2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import json | import json | ||||
import logging | import logging | ||||
import re | import re | ||||
from typing import Iterator, Optional, Sequence, Tuple | from typing import Iterator, Optional, Sequence, Tuple | ||||
import attr | import attr | ||||
from swh.loader.package.loader import BasePackageInfo, PackageLoader | from swh.loader.package.loader import BasePackageInfo, PackageLoader | ||||
from swh.loader.package.utils import EMPTY_AUTHOR, api_info, cached_method, release_name | from swh.loader.package.utils import ( | ||||
EMPTY_AUTHOR, | |||||
get_url_body, | |||||
release_name, | |||||
cached_method, | |||||
) | |||||
from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | from swh.model.model import ObjectType, Release, Sha1Git, TimestampWithTimezone | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
def _uppercase_encode(url: str) -> str: | def _uppercase_encode(url: str) -> str: | ||||
return re.sub("([A-Z]{1})", r"!\1", url).lower() | return re.sub("([A-Z]{1})", r"!\1", url).lower() | ||||
Show All 26 Lines | ): | ||||
assert url.startswith( | assert url.startswith( | ||||
self.GOLANG_PKG_DEV_URL | self.GOLANG_PKG_DEV_URL | ||||
), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ), "Go package URL (%s) not from %s" % (url, self.GOLANG_PKG_DEV_URL) | ||||
self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | self.name = url[len(self.GOLANG_PKG_DEV_URL) + 1 :] | ||||
self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | self.url = url.replace(self.GOLANG_PKG_DEV_URL, self.GOLANG_PROXY_URL) | ||||
self.url = _uppercase_encode(self.url) | self.url = _uppercase_encode(self.url) | ||||
def get_versions(self) -> Sequence[str]: | def get_versions(self) -> Sequence[str]: | ||||
versions = api_info(f"{self.url}/@v/list").decode().splitlines() | versions = get_url_body(f"{self.url}/@v/list").decode().splitlines() | ||||
# some go packages only have a development version not listed by the endpoint above, | # some go packages only have a development version not listed by the endpoint above, | ||||
# so ensure to return it or it will be missed by the golang loader | # so ensure to return it or it will be missed by the golang loader | ||||
default_version = self.get_default_version() | default_version = self.get_default_version() | ||||
if default_version not in versions: | if default_version not in versions: | ||||
versions.append(default_version) | versions.append(default_version) | ||||
return versions | return versions | ||||
@cached_method | @cached_method | ||||
def get_default_version(self) -> str: | def get_default_version(self) -> str: | ||||
latest = api_info(f"{self.url}/@latest") | latest = get_url_body(f"{self.url}/@latest") | ||||
return json.loads(latest)["Version"] | return json.loads(latest)["Version"] | ||||
def _raw_info(self, version: str) -> dict: | def _raw_info(self, version: str) -> dict: | ||||
url = f"{self.url}/@v/{_uppercase_encode(version)}.info" | url = f"{self.url}/@v/{_uppercase_encode(version)}.info" | ||||
return json.loads(api_info(url)) | return json.loads(get_url_body(url)) | ||||
def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | def get_package_info(self, version: str) -> Iterator[Tuple[str, GolangPackageInfo]]: | ||||
# Encode the name because creating nested folders can become problematic | # Encode the name because creating nested folders can become problematic | ||||
encoded_name = self.name.replace("/", "__") | encoded_name = self.name.replace("/", "__") | ||||
filename = f"{encoded_name}-{version}.zip" | filename = f"{encoded_name}-{version}.zip" | ||||
timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | timestamp = TimestampWithTimezone.from_iso8601(self._raw_info(version)["Time"]) | ||||
p_info = GolangPackageInfo( | p_info = GolangPackageInfo( | ||||
url=f"{self.url}/@v/{version}.zip", | url=f"{self.url}/@v/{version}.zip", | ||||
Show All 24 Lines |