Page MenuHomeSoftware Heritage
Files F7123373

D5975.id21529.diff
No OneTemporary
Actions

D5975.id21529.diff
View Options

diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@
loader.deposit=swh.loader.package.deposit:register
loader.nixguix=swh.loader.package.nixguix:register
loader.npm=swh.loader.package.npm:register
+ loader.opam=swh.loader.package.opam:register
loader.pypi=swh.loader.package.pypi:register
""",
classifiers=[
diff --git a/swh/loader/package/opam/__init__.py b/swh/loader/package/opam/__init__.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+from typing import Any, Mapping
+
+
+def register() -> Mapping[str, Any]:
+ """Register the current worker module's definition"""
+ from .loader import OpamLoader
+
+ return {
+ "task_modules": [f"{__name__}.tasks"],
+ "loader": OpamLoader,
+ }
diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/loader.py
@@ -0,0 +1,143 @@
+import io
+import os
+from subprocess import PIPE, Popen, call
+from typing import Iterator, List, Optional, Tuple
+
+import attr
+
+from swh.loader.package.loader import BasePackageInfo, PackageLoader
+from swh.storage.interface import StorageInterface
+
+
+@attr.s
+class OpamPackageInfo(BasePackageInfo):
+ pass
+
+
+class OpamLoader(PackageLoader[OpamPackageInfo]):
+ visit_type = "opam"
+
+ def __init__(
+ self,
+ storage: StorageInterface,
+ url: str,
+ opam_root: str,
+ opam_instance: str,
+ opam_url: str,
+ opam_package: str,
+ max_content_size: Optional[int] = None,
+ ):
+ super().__init__(storage=storage, url=url, max_content_size=max_content_size)
+
+ self.opam_root = opam_root
+ self.opam_instance = opam_instance
+ self.opam_url = opam_url
+ self.opam_package = opam_package
+
+ self.env = os.environ.copy()
+ self.env["OPAMROOT"] = opam_root
+
+ # if there's no opam_root at the expected path, we create a new one
+ # TODO: check that the opam_root is valid ?
+ if not os.path.isdir(opam_root):
+ if os.path.isfile(opam_root):
+ print("wrong opam root")
+ exit(1)
+ else:
+ call(
+ [
+ "opam",
+ "init",
+ "--reinit",
+ "--bare",
+ "--no-setup",
+ opam_instance,
+ opam_url,
+ ],
+ env=self.env,
+ )
+
+ def get_versions(self) -> List[str]:
+ proc = Popen(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "-f",
+ "all-versions",
+ self.opam_package,
+ ],
+ env=self.env,
+ stdout=PIPE,
+ )
+ if proc.stdout is not None:
+ for line in io.TextIOWrapper(proc.stdout):
+ return line.split()
+ # we only care about the first line
+ # and there should be only one line anyway
+ print(
+ f"can't get versions for package {self.opam_package} \
+ (at url {self.url})"
+ )
+ exit(1)
+
+ def get_default_version(self) -> str:
+ proc = Popen(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "-f",
+ "all-versions",
+ self.opam_package,
+ ],
+ env=self.env,
+ stdout=PIPE,
+ )
+ if proc.stdout is not None:
+ for line in io.TextIOWrapper(proc.stdout):
+ # we only care about the first element of the first line
+ # and there should be only one element and one line anyway
+ v = line.split()
+ if len(v) == 1:
+ return v[0]
+ else:
+ break
+ print(
+ f"can't get default version for package {self.opam_package} \
+ (at url {self.url})"
+ )
+ exit(1)
+
+ def get_package_info(
+ self, version: str
+ ) -> Iterator[Tuple[str, OpamPackageInfo]]:
+ proc = Popen(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "-f",
+ "url.src:",
+ f"{self.opam_package}.{version}",
+ ],
+ env=self.env,
+ stdout=PIPE,
+ )
+ if proc.stdout is not None:
+ # filename is optional
+ yield OpamPackageInfo(
+ url="https://...", filename="...-versionX.Y.tar.gz"
+ )
+
+ print(
+ f"can't get url.src for version {version} of package {self.opam_package} \
+ (at url {self.url})"
+ )
+ exit(1)
diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tasks.py
@@ -0,0 +1,14 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from celery import shared_task
+
+from swh.loader.package.opam.loader import OpamLoader
+
+
+@shared_task(name=__name__ + ".LoadOpam")
+def load_opam(url=None, artifacts=[]):
+ """Load Opam's artifacts"""
+ return OpamLoader.from_configfile(url=url, artifacts=artifacts).load()

File Metadata

Mime Type
text/plain
Expires
Thu, Dec 19, 6:51 AM (6 h, 40 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3232055

Event Timeline

About · Developer information · Legal