Page MenuHomeSoftware Heritage
Files F7123380

D5975.id21703.diff
No OneTemporary
Actions

D5975.id21703.diff
View Options

diff --git a/CONTRIBUTORS b/CONTRIBUTORS
--- a/CONTRIBUTORS
+++ b/CONTRIBUTORS
@@ -1 +1,2 @@
-Antoine Eiche
\ No newline at end of file
+Antoine Eiche
+Léo Andrès
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -61,6 +61,7 @@
loader.deposit=swh.loader.package.deposit:register
loader.nixguix=swh.loader.package.nixguix:register
loader.npm=swh.loader.package.npm:register
+ loader.opam=swh.loader.package.opam:register
loader.pypi=swh.loader.package.pypi:register
""",
classifiers=[
diff --git a/swh/loader/package/opam/__init__.py b/swh/loader/package/opam/__init__.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/__init__.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+from typing import Any, Mapping
+
+
+def register() -> Mapping[str, Any]:
+ """Register the current worker module's definition"""
+ from .loader import OpamLoader
+
+ return {
+ "task_modules": [f"{__name__}.tasks"],
+ "loader": OpamLoader,
+ }
diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/loader.py
@@ -0,0 +1,213 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import io
+import os
+from subprocess import PIPE, Popen, call
+from typing import Iterator, List, Optional, Tuple
+
+import attr
+
+from swh.loader.package.loader import BasePackageInfo, PackageLoader
+from swh.model.model import Person, Revision, RevisionType, Sha1Git
+from swh.storage.interface import StorageInterface
+
+
+@attr.s
+class OpamPackageInfo(BasePackageInfo):
+ author = attr.ib(type=Person)
+ committer = attr.ib(type=Person)
+ version = attr.ib(type=str)
+
+
+def opam_read(
+ cmd: List[str], init_error_msg_if_any: Optional[str] = None
+) -> Iterator[str]:
+ """This executes and reads an opam command and yields the
+ output result one line at a time.
+
+ Args:
+ cmd: Opam command to execute as a list of string
+ init_error_msg_if_any: Error message to raise in case a problem occurs
+ during initialization
+
+ Raises:
+ ValueError with the init_error_msg_if_any content in case
+ stdout is not consumable (or something...) and the variable is provided.
+
+ Yields:
+ output line result of the command line
+
+ """
+ with Popen(cmd, stdout=PIPE) as proc:
+ if proc.stdout is not None:
+ for line in io.TextIOWrapper(proc.stdout):
+ yield line
+ elif init_error_msg_if_any:
+ raise ValueError(init_error_msg_if_any)
+
+
+class OpamLoader(PackageLoader[OpamPackageInfo]):
+ """
+ Load all versions of a given package in a given opam repository.
+
+ The state of the opam repository is stored in a directory called an
+ opam root. Either the opam root has been created by the loader and we
+ simply re-use it, either it doesn't exist yet and we create it on the
+ first package we try to load (next packages will be able to re-use it).
+
+ Then we just ask the opam binary to give us the list of all versions of
+ the given package. For each version, we ask the opam binary to give us
+ the url to the tarball to archive.
+ """
+
+ visit_type = "opam"
+
+ def __init__(
+ self,
+ storage: StorageInterface,
+ url: str,
+ opam_root: str,
+ opam_instance: str,
+ opam_url: str,
+ opam_package: str,
+ max_content_size: Optional[int] = None,
+ ):
+ super().__init__(storage=storage, url=url, max_content_size=max_content_size)
+
+ self.opam_root = opam_root
+ self.opam_instance = opam_instance
+ self.opam_url = opam_url
+ self.opam_package = opam_package
+
+ if not os.path.isdir(opam_root):
+ if os.path.isfile(opam_root):
+ raise ValueError("invalid opam root")
+ else:
+ call(
+ [
+ "opam",
+ "init",
+ "--reinit",
+ "--bare",
+ "--no-setup",
+ "--root",
+ opam_root,
+ opam_instance,
+ opam_url,
+ ]
+ )
+ elif not os.path.isfile(os.path.join(opam_root, "config")):
+ raise ValueError("invalid opam root")
+
+ def get_versions(self) -> List[str]:
+ init_error_msg = f"can't get versions for package {self.opam_package} \
+ (at url {self.url}) from `opam show`"
+ for line in opam_read(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "--root",
+ self.opam_root,
+ "-f",
+ "all-versions",
+ self.opam_package,
+ ],
+ init_error_msg_if_any=init_error_msg,
+ ):
+ # only care about the first and only line which hold the
+ # versions information as a blank separated list
+ return line.split()
+ raise ValueError(init_error_msg)
+
+ def get_default_version(self) -> str:
+
+ init_error_msg = f"can't get default version for package {self.opam_package} \
+ (at url {self.url}) from `opam show`"
+ for line in opam_read(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "--root",
+ self.opam_root,
+ "-f",
+ "version",
+ self.opam_package,
+ ],
+ init_error_msg_if_any=init_error_msg,
+ ):
+ # we only care about the first element of the first line
+ # and there should be only one element and one line anyway
+ v = line.split()
+ if len(v) != 1:
+ raise ValueError(init_error_msg)
+ return v[0]
+ raise ValueError(init_error_msg)
+
+ def get_enclosed_single_line_field(self, field, version) -> Optional[str]:
+ for line in opam_read(
+ [
+ "opam",
+ "show",
+ "--color",
+ "never",
+ "--normalise",
+ "--root",
+ self.opam_root,
+ "-f",
+ field,
+ f"{self.opam_package}.{version}",
+ ]
+ ):
+ # we only care about the first line
+ # and there should be only one line anyway
+ # we also need to remove the enclosing " and the trailing \n
+ return line[1:-2]
+ return None
+
+ def get_package_info(self, version: str) -> Iterator[Tuple[str, OpamPackageInfo]]:
+
+ branch_name = f"{self.opam_package}.{version}"
+ url = self.get_enclosed_single_line_field("url.src:", version)
+
+ if url is None:
+ raise ValueError(
+ f"can't get field url.src: for version {version} of package {self.opam_package} \
+ (at url {self.url}) from `opam show`"
+ )
+
+ authors_field = self.get_enclosed_single_line_field("authors:", version)
+ fullname = b"" if authors_field is None else str.encode(authors_field)
+ author = Person(fullname=fullname, name=None, email=None)
+
+ maintainer_field = self.get_enclosed_single_line_field("maintainer:", version)
+ fullname = b"" if maintainer_field is None else str.encode(maintainer_field)
+ committer = Person(fullname=fullname, name=None, email=None)
+
+ yield branch_name, OpamPackageInfo(
+ url=url, filename=None, author=author, committer=committer, version=version
+ )
+
+ def build_revision(
+ self, p_info: OpamPackageInfo, uncompressed_path: str, directory: Sha1Git
+ ) -> Optional[Revision]:
+
+ return Revision(
+ type=RevisionType.TAR,
+ author=p_info.author,
+ committer=p_info.committer,
+ message=str.encode(p_info.version),
+ date=None,
+ committer_date=None,
+ parents=(),
+ directory=directory,
+ synthetic=True,
+ )
diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tasks.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from celery import shared_task
+
+from swh.loader.package.opam.loader import OpamLoader
+
+
+@shared_task(name=__name__ + ".LoadOpam")
+def load_opam(url, opam_root, opam_instance, opam_url, opam_package):
+ """Load Opam's artifacts"""
+ return OpamLoader.from_configfile(
+ url=url,
+ opam_root=opam_root,
+ opam_instance=opam_instance,
+ opam_url=opam_url,
+ opam_package=opam_package,
+ ).load()
diff --git a/swh/loader/package/opam/tests/__init__.py b/swh/loader/package/opam/tests/__init__.py
new file mode 100644
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam
@@ -0,0 +1,38 @@
+opam-version: "2.0"
+synopsis: "Adjustable grid (two dimensional array) library"
+description:
+ "Adjustable grids are two dimensional arrays whose width/height can be changed by adding or removing row/column at either end (one at a time)."
+maintainer: ["OCamlPro <contact@ocamlpro.com>"]
+authors: ["OCamlPro <contact@ocamlpro.com>"]
+license: "ISC"
+homepage: "https://github.com/ocamlpro/agrid"
+bug-reports: "https://github.com/ocamlpro/agrid/issues"
+depends: [
+ "ocaml" {>= "4.05"}
+ "dune" {>= "2.7"}
+ "flex-array" {>= "1.2"}
+ "bisect_ppx" {with-test & >= "2.6" & dev}
+ "odoc" {with-doc}
+]
+build: [
+ ["dune" "subst"] {dev}
+ [
+ "dune"
+ "build"
+ "-p"
+ name
+ "-j"
+ jobs
+ "@install"
+ "@runtest" {with-test}
+ "@doc" {with-doc}
+ ]
+]
+dev-repo: "git+https://github.com/ocamlpro/agrid.git"
+url {
+ src: "https://github.com/OCamlPro/agrid/archive/0.1.tar.gz"
+ checksum: [
+ "sha256=ea82546711a6abdd4edf8bc3052041498cae9c2e5a9e147e29820da4eac4beb4"
+ "sha512=f53b2c095e3607e53f92d4e7e13848e9e34bd866837335e7d9341dbb468ac46ffbcd2002d1bf1105e2f6060f57871aef7ce8e65594855447fafb72ad32b076b7"
+ ]
+}
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam
@@ -0,0 +1,38 @@
+opam-version: "2.0"
+synopsis:
+ "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
+description:
+ "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS."
+maintainer: ["OCamlPro <contact@ocamlpro.com>"]
+authors: ["OCamlPro <contact@ocamlpro.com>"]
+license: "ISC"
+homepage: "https://github.com/ocamlpro/directories"
+bug-reports: "https://github.com/ocamlpro/directories/issues"
+depends: [
+ "dune" {>= "2.1"}
+ "ocaml" {>= "4.07.0"}
+ "ctypes-foreign" {>= "0.4.0" | "os" != "win32" | "os" != "mingw"}
+ "ctypes" {>= "0.17.1" | "os" != "win32" | "os" != "mingw"}
+]
+build: [
+ ["dune" "subst"] {dev}
+ [
+ "dune"
+ "build"
+ "-p"
+ name
+ "-j"
+ jobs
+ "@install"
+ "@runtest" {with-test}
+ "@doc" {with-doc}
+ ]
+]
+dev-repo: "git+https://github.com/ocamlpro/directories.git"
+url {
+ src: "https://github.com/OCamlPro/directories/archive/0.1.tar.gz"
+ checksum: [
+ "sha256=89aa3586af3c38aea17302cfb0b6243afdf0fd90b2eb1f86fb69990917304445"
+ "sha512=3302f8c8c2c3ecc217f199e97d8d32d1ef14af9dd3763cbc1030522832ce375192422f7b6b6acd8cd8399d96ccd52d5e5cfffac3cb392f1cf8b9f67374c1cf7c"
+ ]
+}
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam
@@ -0,0 +1,38 @@
+opam-version: "2.0"
+synopsis:
+ "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
+description:
+ "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS."
+maintainer: ["OCamlPro <contact@ocamlpro.com>"]
+authors: ["OCamlPro <contact@ocamlpro.com>"]
+license: "ISC"
+homepage: "https://github.com/ocamlpro/directories"
+bug-reports: "https://github.com/ocamlpro/directories/issues"
+depends: [
+ "dune" {>= "2.1"}
+ "ocaml" {>= "4.07.0"}
+ "ctypes-foreign" {>= "0.4.0" & (os = "win32" | os = "cygwin")}
+ "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")}
+]
+build: [
+ ["dune" "subst"] {dev}
+ [
+ "dune"
+ "build"
+ "-p"
+ name
+ "-j"
+ jobs
+ "@install"
+ "@runtest" {with-test}
+ "@doc" {with-doc}
+ ]
+]
+dev-repo: "git+https://github.com/ocamlpro/directories.git"
+url {
+ src: "https://github.com/OCamlPro/directories/archive/0.2.tar.gz"
+ checksum: [
+ "sha256=af81e7bd7dd7125eb5168226def47e6e2287f3210ba33c015b58123a1877c40d"
+ "sha512=30c0db12eb453f3549bf52c202d6e6a8d12e8f4f07b92d8fc68309beb4dcf66b610eab49ac4bd3e1ddd487cd1539790c92f9511c5c06d33c2bf966148c08aa82"
+ ]
+}
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam
@@ -0,0 +1,43 @@
+opam-version: "2.0"
+synopsis:
+ "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows"
+description: """
+directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows.
+It is inspired by similar libraries for other languages such as directories-jvm.
+The following conventions are used:
+- XDG Base Directory Specification and xdg-user-dirs on Linux
+- Known Folders on Windows
+- Standard Directories on macOS.
+"""
+maintainer: ["OCamlPro <contact@ocamlpro.com>"]
+authors: ["OCamlPro <contact@ocamlpro.com>"]
+license: "ISC"
+homepage: "https://github.com/ocamlpro/directories"
+bug-reports: "https://github.com/ocamlpro/directories/issues"
+depends: [
+ "dune" {>= "2.1"}
+ "ocaml" {>= "4.07.0"}
+ "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")}
+]
+build: [
+ ["dune" "subst"] {dev}
+ [
+ "dune"
+ "build"
+ "-p"
+ name
+ "-j"
+ jobs
+ "@install"
+ "@runtest" {with-test}
+ "@doc" {with-doc}
+ ]
+]
+dev-repo: "git+https://github.com/ocamlpro/directories.git"
+url {
+ src: "https://github.com/OCamlPro/directories/archive/0.3.tar.gz"
+ checksum: [
+ "sha256=9b37d1d43e3b06f3b68ebe57651ea2cf5a9db6d5bdb1a0afe65786bcdcc8bf11"
+ "sha512=9d1634a0c44dd74dc3005154ebbc22c39bb0c3e1477f3e67c9fc49cc32059ac0f496816c06785f204f60ee67136c998f526ba0c9bf4232d040ff30bd89411fb3"
+ ]
+}
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam
@@ -0,0 +1,37 @@
+opam-version: "2.0"
+synopsis: "SVG badge generator"
+description:
+ "An OCaml library for SVG badge generation. There's also a command-line tool provided."
+maintainer: ["OCamlPro <contact@ocamlpro.com>"]
+authors: ["OCamlPro <contact@ocamlpro.com>"]
+license: "ISC"
+homepage: "https://ocamlpro.github.io/ocb/"
+doc: "https://ocamlpro.github.io/ocb/api/"
+bug-reports: "https://github.com/OCamlPro/ocb/issues"
+depends: [
+ "ocaml" {>= "4.05"}
+ "dune" {>= "2.0"}
+ "odoc" {with-doc}
+]
+build: [
+ ["dune" "subst"] {dev}
+ [
+ "dune"
+ "build"
+ "-p"
+ name
+ "-j"
+ jobs
+ "@install"
+ "@runtest" {with-test}
+ "@doc" {with-doc}
+ ]
+]
+dev-repo: "git+https://github.com/OCamlPro/ocb.git"
+url {
+ src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz"
+ checksum: [
+ "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46"
+ "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b"
+ ]
+}
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/repo b/swh/loader/package/opam/tests/data/fake_opam_repo/repo
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/repo
@@ -0,0 +1 @@
+opam-version: "2.0"
diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/version b/swh/loader/package/opam/tests/data/fake_opam_repo/version
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/data/fake_opam_repo/version
@@ -0,0 +1 @@
+0.9.0
diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_agrid_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_agrid_archive_0.1.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.1.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.2.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.2.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.3.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.3.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_ocb_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_ocb_archive_0.1.tar.gz
new file mode 100644
index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000
GIT binary patch
literal 0
Hc$@<O00001
literal 0
Hc$@<O00001
diff --git a/swh/loader/package/opam/tests/test_opam.py b/swh/loader/package/opam/tests/test_opam.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/test_opam.py
@@ -0,0 +1,176 @@
+# Copyright (C) 2019-2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+from shutil import rmtree
+
+from swh.loader.package.opam.loader import OpamLoader, OpamPackageInfo
+from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats
+from swh.model.hashutil import hash_to_bytes
+from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType
+
+
+def test_opam_loader_one_version(tmpdir, requests_mock_datadir, datadir, swh_storage):
+
+ opam_url = f"file://{datadir}/fake_opam_repo"
+
+ opam_root = tmpdir
+ # the directory should NOT exist, we just need an unique name, so we delete it
+ rmtree(tmpdir)
+
+ opam_instance = "loadertest"
+
+ opam_package = "agrid"
+ url = f"opam+{opam_url}/packages/{opam_package}"
+
+ loader = OpamLoader(
+ swh_storage, url, opam_root, opam_instance, opam_url, opam_package
+ )
+
+ actual_load_status = loader.load()
+
+ expected_snapshot_id = hash_to_bytes("4e4bf977312460329d7f769b0be89937c9827efc")
+ assert actual_load_status == {
+ "status": "eventful",
+ "snapshot_id": expected_snapshot_id.hex(),
+ }
+
+ target = b"S\x8c\x8aq\xdcy\xa4/0\xa0\xb2j\xeb\xc1\x16\xad\xce\x06\xeaV"
+
+ expected_snapshot = Snapshot(
+ id=expected_snapshot_id,
+ branches={
+ b"HEAD": SnapshotBranch(target=b"agrid.0.1", target_type=TargetType.ALIAS,),
+ b"agrid.0.1": SnapshotBranch(
+ target=target, target_type=TargetType.REVISION,
+ ),
+ },
+ )
+ check_snapshot(expected_snapshot, swh_storage)
+
+ assert_last_visit_matches(
+ swh_storage, url, status="full", type="opam", snapshot=expected_snapshot_id
+ )
+
+ stats = get_stats(swh_storage)
+
+ assert {
+ "content": 18,
+ "directory": 8,
+ "origin": 1,
+ "origin_visit": 1,
+ "release": 0,
+ "revision": 1,
+ "skipped_content": 0,
+ "snapshot": 1,
+ } == stats
+
+
+def test_opam_loader_many_version(tmpdir, requests_mock_datadir, datadir, swh_storage):
+
+ opam_url = f"file://{datadir}/fake_opam_repo"
+
+ opam_root = tmpdir
+ # the directory should NOT exist, we just need an unique name, so we delete it
+ rmtree(tmpdir)
+
+ opam_instance = "loadertest"
+
+ opam_package = "directories"
+ url = f"opam+{opam_url}/packages/{opam_package}"
+
+ loader = OpamLoader(
+ swh_storage, url, opam_root, opam_instance, opam_url, opam_package
+ )
+
+ actual_load_status = loader.load()
+
+ expected_snapshot_id = hash_to_bytes("1b49be175dcf17c0f568bcd7aac3d4faadc41249")
+ assert actual_load_status == {
+ "status": "eventful",
+ "snapshot_id": expected_snapshot_id.hex(),
+ }
+
+ expected_snapshot = Snapshot(
+ id=expected_snapshot_id,
+ branches={
+ b"HEAD": SnapshotBranch(
+ target=b"directories.0.3", target_type=TargetType.ALIAS,
+ ),
+ b"directories.0.1": SnapshotBranch(
+ target=b"N\x92jA\xb2\x892\xeb\xcc\x9c\xa9\xb3\xea\xa7kz\xb08\xa6V",
+ target_type=TargetType.REVISION,
+ ),
+ b"directories.0.2": SnapshotBranch(
+ target=b"yj\xc9\x1a\x8f\xe0\xaa\xff[\x88\xffz"
+ b"\x91C\xcc\x96\xb7\xd4\xf65",
+ target_type=TargetType.REVISION,
+ ),
+ b"directories.0.3": SnapshotBranch(
+ target=b"hA \xc4\xb5\x18A8\xb8C\x12\xa3\xa5T\xb7/v\x85X\xcb",
+ target_type=TargetType.REVISION,
+ ),
+ },
+ )
+
+ check_snapshot(expected_snapshot, swh_storage)
+
+ assert_last_visit_matches(
+ swh_storage, url, status="full", type="opam", snapshot=expected_snapshot_id
+ )
+
+
+def test_opam_revision(tmpdir, requests_mock_datadir, swh_storage, datadir):
+
+ opam_url = f"file://{datadir}/fake_opam_repo"
+
+ opam_root = tmpdir
+ # the directory should NOT exist, we just need an unique name, so we delete it
+ rmtree(tmpdir)
+
+ opam_instance = "loadertest"
+
+ opam_package = "ocb"
+ url = f"opam+{opam_url}/packages/{opam_package}"
+
+ loader = OpamLoader(
+ swh_storage, url, opam_root, opam_instance, opam_url, opam_package
+ )
+
+ actual_load_status = loader.load()
+
+ expected_snapshot_id = hash_to_bytes("398df115b9feb2f463efd21941d69b7d59cd9025")
+ assert actual_load_status == {
+ "status": "eventful",
+ "snapshot_id": expected_snapshot_id.hex(),
+ }
+
+ info_iter = loader.get_package_info("0.1")
+ branch_name, package_info = next(info_iter)
+ expected_branch_name = "ocb.0.1"
+ expected_package_info = OpamPackageInfo(
+ url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz",
+ filename=None,
+ directory_extrinsic_metadata=[],
+ author=Person(
+ fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
+ ),
+ committer=Person(
+ fullname=b"OCamlPro <contact@ocamlpro.com>", name=None, email=None
+ ),
+ version="0.1",
+ )
+
+ assert branch_name == expected_branch_name
+ assert package_info == expected_package_info
+
+ revision_id = b"o\xad\x7f=\x07\xbb\xaah\xdbI(\xb0'\x10z\xfc\xff\x06x\x1b"
+
+ revision = swh_storage.revision_get([revision_id])[0]
+
+ assert revision is not None
+
+ assert revision.author == expected_package_info.author
+ assert revision.committer == expected_package_info.committer
diff --git a/swh/loader/package/opam/tests/test_tasks.py b/swh/loader/package/opam/tests/test_tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/package/opam/tests/test_tasks.py
@@ -0,0 +1,27 @@
+# Copyright (C) 2019-2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def test_tasks_opam_loader(
+ mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config
+):
+ mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load")
+ mock_load.return_value = {"status": "eventful"}
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.package.opam.tasks.LoadOpam",
+ args=(
+ "opam+https://opam.ocaml.org/packages/agrid", # url
+ "/tmp/test_tasks_opam_loader", # opam_root
+ "test_tasks_opam_loader", # opam_instance
+ "https://opam.ocaml.org", # opam_url
+ "agrid", # opam_package
+ ),
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+ assert mock_load.called
+ assert res.result == {"status": "eventful"}

File Metadata

Mime Type
text/plain
Expires
Thu, Dec 19, 7:01 AM (13 h, 21 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3229916

Event Timeline

About · Developer information · Legal