diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -61,6 +61,7 @@ loader.deposit=swh.loader.package.deposit:register loader.nixguix=swh.loader.package.nixguix:register loader.npm=swh.loader.package.npm:register + loader.opam=swh.loader.package.opam:register loader.pypi=swh.loader.package.pypi:register """, classifiers=[ diff --git a/swh/loader/package/opam/__init__.py b/swh/loader/package/opam/__init__.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/opam/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from .loader import OpamLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": OpamLoader, + } diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/opam/loader.py @@ -0,0 +1,142 @@ +import io +import os +from subprocess import PIPE, Popen, call +from typing import Iterator, List, Optional, Tuple + +import attr + +from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.storage.interface import StorageInterface + + +@attr.s +class OpamPackageInfo(BasePackageInfo): + pass + + +class OpamLoader(PackageLoader[OpamPackageInfo]): + visit_type = "opam" + + def __init__( + self, + storage: StorageInterface, + url: str, + opam_root: str, + opam_instance: str, + opam_url: str, + opam_package: str, + max_content_size: Optional[int] = None, + ): + super().__init__(storage=storage, url=url, max_content_size=max_content_size) + + self.opam_root = opam_root + self.opam_instance = opam_instance + self.opam_url = opam_url + self.opam_package = opam_package + + self.env = os.environ.copy() + self.env["OPAMROOT"] = opam_root + + # if there's no opam_root at the expected path, we create a new one + # TODO: check that the opam_root is valid ? + if not os.path.isdir(opam_root): + if os.path.isfile(opam_root): + print("wrong opam root") + exit(1) + else: + call( + [ + "opam", + "init", + "--reinit", + "--bare", + "--no-setup", + opam_instance, + opam_url, + ], + env=self.env, + ) + + def get_versions(self) -> List[str]: + proc = Popen( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "-f", + "all-versions", + self.opam_package, + ], + env=self.env, + stdout=PIPE, + ) + if proc.stdout is not None: + for line in io.TextIOWrapper(proc.stdout): + return line.split() + # we only care about the first line + # and there should be only one line anyway + print( + f"can't get versions for package {self.opam_package} \ + (at url {self.url})" + ) + exit(1) + + def get_default_version(self) -> str: + proc = Popen( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "-f", + "all-versions", + self.opam_package, + ], + env=self.env, + stdout=PIPE, + ) + if proc.stdout is not None: + for line in io.TextIOWrapper(proc.stdout): + # we only care about the first element of the first line + # and there should be only one element and one line anyway + if len(line) == 1: + return line.split()[0] + else: + break + print( + f"can't get default version for package {self.opam_package} \ + (at url {self.url})" + ) + exit(1) + + def get_package_info( + self, version: str + ) -> Iterator[Tuple[str, OpamPackageInfo]]: + proc = Popen( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "-f", + "url.src:", + f"{self.opam_package}.{version}", + ], + env=self.env, + stdout=PIPE, + ) + if proc.stdout is not None: + # filename is optional + yield OpamPackageInfo( + url="https://...", filename="...-versionX.Y.tar.gz" + ) + + print( + f"can't get url.src for version {version} of package {self.opam_package} \ + (at url {self.url})" + ) + exit(1) diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/opam/tasks.py @@ -0,0 +1,14 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.package.opam.loader import OpamLoader + + +@shared_task(name=__name__ + ".LoadOpam") +def load_opam(url=None, artifacts=[]): + """Load Opam's artifacts""" + return OpamLoader.from_configfile(url=url, artifacts=artifacts).load()