Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/package/arch/loader.py
- This file was added.
| # Copyright (C) 2022 The Software Heritage developers | |||||||||||||
| # See the AUTHORS file at the top-level directory of this distribution | |||||||||||||
| # License: GNU General Public License version 3, or any later version | |||||||||||||
| # See top-level LICENSE file for more information | |||||||||||||
| from distutils.version import LooseVersion | |||||||||||||
| from pathlib import Path | |||||||||||||
| import re | |||||||||||||
| from typing import Any, Dict, Iterator, List, Optional, Sequence, Tuple | |||||||||||||
| import attr | |||||||||||||
| from swh.loader.package.loader import BasePackageInfo, PackageLoader | |||||||||||||
| from swh.loader.package.utils import release_name | |||||||||||||
| from swh.model.model import ObjectType, Person, Release, Sha1Git, TimestampWithTimezone | |||||||||||||
| from swh.storage.interface import StorageInterface | |||||||||||||
| @attr.s | |||||||||||||
| class ArchPackageInfo(BasePackageInfo): | |||||||||||||
| name = attr.ib(type=str) | |||||||||||||
| """Name of the package""" | |||||||||||||
| version = attr.ib(type=str) | |||||||||||||
| """Current version""" | |||||||||||||
| last_modified = attr.ib(type=str) | |||||||||||||
| """File last modified date as release date""" | |||||||||||||
| def extract_intrinsic_metadata(dir_path: Path) -> Dict[str, Any]: | |||||||||||||
| """Extract intrinsic metadata from .PKGINFO file at dir_path. | |||||||||||||
| Each Arch linux package has a .PKGINFO file at the root of the archive. | |||||||||||||
| Args: | |||||||||||||
| dir_path: A directory on disk where a package has been extracted | |||||||||||||
| Returns: | |||||||||||||
| A dict mapping | |||||||||||||
| """ | |||||||||||||
| pkginfo_path = Path(dir_path, ".PKGINFO") | |||||||||||||
| rex = re.compile(r"^(\w+)\s=\s(.*)$", re.M) | |||||||||||||
| with pkginfo_path.open("rb") as content: | |||||||||||||
| parsed = rex.findall(content.read().decode()) | |||||||||||||
| data = {entry[0].lower(): entry[1] for entry in parsed} | |||||||||||||
| if "url" in data.keys(): | |||||||||||||
| data["project_url"] = data["url"] | |||||||||||||
| return data | |||||||||||||
| class ArchLoader(PackageLoader[ArchPackageInfo]): | |||||||||||||
| visit_type = "arch" | |||||||||||||
| def __init__( | |||||||||||||
| self, | |||||||||||||
| storage: StorageInterface, | |||||||||||||
| url: str, | |||||||||||||
| artifacts: List[Dict[str, Any]], | |||||||||||||
| **kwargs, | |||||||||||||
| ): | |||||||||||||
| super().__init__(storage=storage, url=url, **kwargs) | |||||||||||||
ardumont: with:
```
from ..pattern import CredentialsType
from typing import Optional
```
for the types… | |||||||||||||
Not Done Inline Actionswait, nvm, we are in package loader... ardumont: wait, nvm, we are in package loader...
sorry for the hit and miss ;) | |||||||||||||
Done Inline Actions;-) franckbret: ;-) | |||||||||||||
Done Inline Actions
even better ^ that's what we decided to do with all loaders now vlorentz: even better ^
that's what we decided to do with all loaders now | |||||||||||||
Done Inline Actions+1 ^ ardumont: +1 ^ | |||||||||||||
| self.url = url | |||||||||||||
| self.artifacts: Dict[str, Dict] = { | |||||||||||||
| artifact["version"]: artifact for artifact in artifacts | |||||||||||||
| } | |||||||||||||
| def get_versions(self) -> Sequence[str]: | |||||||||||||
| """Get all released versions of an Arch Linux package | |||||||||||||
| Returns: | |||||||||||||
| A sequence of versions | |||||||||||||
| Example:: | |||||||||||||
| ["0.1.1", "0.10.2"] | |||||||||||||
| """ | |||||||||||||
| versions = list(self.artifacts.keys()) | |||||||||||||
| versions.sort(key=LooseVersion) | |||||||||||||
| return versions | |||||||||||||
| def get_default_version(self) -> str: | |||||||||||||
| """Get the newest release version of an Arch Linux package | |||||||||||||
| Returns: | |||||||||||||
| A string representing a version | |||||||||||||
| Example:: | |||||||||||||
| "0.1.2" | |||||||||||||
| """ | |||||||||||||
| return self.get_versions()[-1] | |||||||||||||
| def get_package_info(self, version: str) -> Iterator[Tuple[str, ArchPackageInfo]]: | |||||||||||||
| """Get release name and package information from version | |||||||||||||
| Args: | |||||||||||||
| version: arch version (e.g: "0.1.0") | |||||||||||||
| Returns: | |||||||||||||
| Iterator of tuple (release_name, p_info) | |||||||||||||
| """ | |||||||||||||
| artifact = self.artifacts[version] | |||||||||||||
Done Inline Actions
To ensure we dot not wrongly assume too much. ardumont: To ensure we dot not wrongly assume too much. | |||||||||||||
| assert version == artifact["version"] | |||||||||||||
| p_info = ArchPackageInfo( | |||||||||||||
| name=artifact["name"], | |||||||||||||
| filename=artifact["filename"], | |||||||||||||
| url=artifact["url"], | |||||||||||||
| version=version, | |||||||||||||
| last_modified=artifact["last_modified"], | |||||||||||||
| ) | |||||||||||||
| yield release_name(version, artifact["filename"]), p_info | |||||||||||||
| def build_release( | |||||||||||||
| self, p_info: ArchPackageInfo, uncompressed_path: str, directory: Sha1Git | |||||||||||||
| ) -> Optional[Release]: | |||||||||||||
Done Inline Actionsfranckbret: @ardumont @vlorentz Does it add any value to add intrinsic_metadata to p_info object? | |||||||||||||
Not Done Inline ActionsThey are more readily available when in the release message right now, because metadata views are really bad. vlorentz: They are more readily available when in the release message right now, because metadata views… | |||||||||||||
| intrinsic_metadata = extract_intrinsic_metadata(Path(uncompressed_path)) | |||||||||||||
| author = Person.from_fullname(intrinsic_metadata["packager"].encode()) | |||||||||||||
| description = intrinsic_metadata["pkgdesc"] | |||||||||||||
| message = ( | |||||||||||||
| f"Synthetic release for Arch Linux source package {p_info.name} " | |||||||||||||
| f"version {p_info.version}\n\n" | |||||||||||||
Not Done Inline Actions
for consistency with D7998 vlorentz: for consistency with D7998 | |||||||||||||
Done Inline ActionsOk Do you think it valuable to add in this arch message {p.arch} and {p.repo}? franckbret: Ok
Do you think it valuable to add in this arch message {p.arch} and {p.repo}? | |||||||||||||
Done Inline ActionsIt does not hurt. Although, if you look at the exising loader packages [1], we limit the @vlorentz any more thoughts on the matter? ardumont: It does not hurt. Although, if you look at the exising loader packages [1], we limit the… | |||||||||||||
Not Done Inline ActionsHmm, I don't know. But if we remove them here, then we should remove them from the Crates loader as well. What do the descriptions look like in practice, for Arch and Crates? vlorentz: Hmm, I don't know. But if we remove them here, then we should remove them from the Crates… | |||||||||||||
Done Inline Actionsto clarify, i was just saying "keep the actual implementation" to keep the crafted message minimalistic like the other package loader and not add the suggested p.arch and p.repo. ardumont: to clarify, i was just saying "keep the actual implementation" to keep the crafted message… | |||||||||||||
| f"{description}\n" | |||||||||||||
| ) | |||||||||||||
| return Release( | |||||||||||||
| name=p_info.version.encode(), | |||||||||||||
| author=author, | |||||||||||||
| date=TimestampWithTimezone.from_iso8601(p_info.last_modified), | |||||||||||||
| message=message.encode(), | |||||||||||||
| target_type=ObjectType.DIRECTORY, | |||||||||||||
| target=directory, | |||||||||||||
| synthetic=True, | |||||||||||||
Done Inline Actionsvlorentz: please add this to https://forge.softwareheritage.org/source/swh-loader… | |||||||||||||
Done Inline Actionsthat will show up in the doc when we land and tag [1] ardumont: that will show up in the doc when we land and tag [1]
[1] https://docs.softwareheritage. | |||||||||||||
| ) | |||||||||||||
with:
for the types
I recall that breaks otherwise when deployed.