diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 48d0962..bcf34f8 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1 +1,2 @@ -Antoine Eiche \ No newline at end of file +Antoine Eiche +Léo Andrès diff --git a/setup.py b/setup.py index 0896b3d..ce13550 100755 --- a/setup.py +++ b/setup.py @@ -1,79 +1,80 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.rst"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.loader.core", description="Software Heritage Base Loader", long_description=long_description, long_description_content_type="text/x-rst", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DLDBASE", packages=find_packages(), # packages's modules scripts=[], # scripts to package install_requires=parse_requirements() + parse_requirements("swh"), setup_requires=["setuptools-scm"], use_scm_version=True, extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" [swh.cli.subcommands] loader=swh.loader.cli [swh.workers] loader.archive=swh.loader.package.archive:register loader.cran=swh.loader.package.cran:register loader.debian=swh.loader.package.debian:register loader.deposit=swh.loader.package.deposit:register loader.nixguix=swh.loader.package.nixguix:register loader.npm=swh.loader.package.npm:register + loader.opam=swh.loader.package.opam:register loader.pypi=swh.loader.package.pypi:register """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-loader-core", "Documentation": "https://docs.softwareheritage.org/devel/swh-loader-core/", }, ) diff --git a/swh/loader/package/opam/__init__.py b/swh/loader/package/opam/__init__.py new file mode 100644 index 0000000..ab06817 --- /dev/null +++ b/swh/loader/package/opam/__init__.py @@ -0,0 +1,17 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from .loader import OpamLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": OpamLoader, + } diff --git a/swh/loader/package/opam/loader.py b/swh/loader/package/opam/loader.py new file mode 100644 index 0000000..7e55ca0 --- /dev/null +++ b/swh/loader/package/opam/loader.py @@ -0,0 +1,213 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import io +import os +from subprocess import PIPE, Popen, call +from typing import Iterator, List, Optional, Tuple + +import attr + +from swh.loader.package.loader import BasePackageInfo, PackageLoader +from swh.model.model import Person, Revision, RevisionType, Sha1Git +from swh.storage.interface import StorageInterface + + +@attr.s +class OpamPackageInfo(BasePackageInfo): + author = attr.ib(type=Person) + committer = attr.ib(type=Person) + version = attr.ib(type=str) + + +def opam_read( + cmd: List[str], init_error_msg_if_any: Optional[str] = None +) -> Iterator[str]: + """This executes and reads an opam command and yields the + output result one line at a time. + + Args: + cmd: Opam command to execute as a list of string + init_error_msg_if_any: Error message to raise in case a problem occurs + during initialization + + Raises: + ValueError with the init_error_msg_if_any content in case + stdout is not consumable (or something...) and the variable is provided. + + Yields: + output line result of the command line + + """ + with Popen(cmd, stdout=PIPE) as proc: + if proc.stdout is not None: + for line in io.TextIOWrapper(proc.stdout): + yield line + elif init_error_msg_if_any: + raise ValueError(init_error_msg_if_any) + + +class OpamLoader(PackageLoader[OpamPackageInfo]): + """ + Load all versions of a given package in a given opam repository. + + The state of the opam repository is stored in a directory called an + opam root. Either the opam root has been created by the loader and we + simply re-use it, either it doesn't exist yet and we create it on the + first package we try to load (next packages will be able to re-use it). + + Then we just ask the opam binary to give us the list of all versions of + the given package. For each version, we ask the opam binary to give us + the url to the tarball to archive. + """ + + visit_type = "opam" + + def __init__( + self, + storage: StorageInterface, + url: str, + opam_root: str, + opam_instance: str, + opam_url: str, + opam_package: str, + max_content_size: Optional[int] = None, + ): + super().__init__(storage=storage, url=url, max_content_size=max_content_size) + + self.opam_root = opam_root + self.opam_instance = opam_instance + self.opam_url = opam_url + self.opam_package = opam_package + + if not os.path.isdir(opam_root): + if os.path.isfile(opam_root): + raise ValueError("invalid opam root") + else: + call( + [ + "opam", + "init", + "--reinit", + "--bare", + "--no-setup", + "--root", + opam_root, + opam_instance, + opam_url, + ] + ) + elif not os.path.isfile(os.path.join(opam_root, "config")): + raise ValueError("invalid opam root") + + def get_versions(self) -> List[str]: + init_error_msg = f"can't get versions for package {self.opam_package} \ + (at url {self.url}) from `opam show`" + for line in opam_read( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "--root", + self.opam_root, + "-f", + "all-versions", + self.opam_package, + ], + init_error_msg_if_any=init_error_msg, + ): + # only care about the first and only line which hold the + # versions information as a blank separated list + return line.split() + raise ValueError(init_error_msg) + + def get_default_version(self) -> str: + + init_error_msg = f"can't get default version for package {self.opam_package} \ + (at url {self.url}) from `opam show`" + for line in opam_read( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "--root", + self.opam_root, + "-f", + "version", + self.opam_package, + ], + init_error_msg_if_any=init_error_msg, + ): + # we only care about the first element of the first line + # and there should be only one element and one line anyway + v = line.split() + if len(v) != 1: + raise ValueError(init_error_msg) + return v[0] + raise ValueError(init_error_msg) + + def get_enclosed_single_line_field(self, field, version) -> Optional[str]: + for line in opam_read( + [ + "opam", + "show", + "--color", + "never", + "--normalise", + "--root", + self.opam_root, + "-f", + field, + f"{self.opam_package}.{version}", + ] + ): + # we only care about the first line + # and there should be only one line anyway + # we also need to remove the enclosing " and the trailing \n + return line[1:-2] + return None + + def get_package_info(self, version: str) -> Iterator[Tuple[str, OpamPackageInfo]]: + + branch_name = f"{self.opam_package}.{version}" + url = self.get_enclosed_single_line_field("url.src:", version) + + if url is None: + raise ValueError( + f"can't get field url.src: for version {version} of package {self.opam_package} \ + (at url {self.url}) from `opam show`" + ) + + authors_field = self.get_enclosed_single_line_field("authors:", version) + fullname = b"" if authors_field is None else str.encode(authors_field) + author = Person(fullname=fullname, name=None, email=None) + + maintainer_field = self.get_enclosed_single_line_field("maintainer:", version) + fullname = b"" if maintainer_field is None else str.encode(maintainer_field) + committer = Person(fullname=fullname, name=None, email=None) + + yield branch_name, OpamPackageInfo( + url=url, filename=None, author=author, committer=committer, version=version + ) + + def build_revision( + self, p_info: OpamPackageInfo, uncompressed_path: str, directory: Sha1Git + ) -> Optional[Revision]: + + return Revision( + type=RevisionType.TAR, + author=p_info.author, + committer=p_info.committer, + message=str.encode(p_info.version), + date=None, + committer_date=None, + parents=(), + directory=directory, + synthetic=True, + ) diff --git a/swh/loader/package/opam/tasks.py b/swh/loader/package/opam/tasks.py new file mode 100644 index 0000000..2ca7f7a --- /dev/null +++ b/swh/loader/package/opam/tasks.py @@ -0,0 +1,20 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.package.opam.loader import OpamLoader + + +@shared_task(name=__name__ + ".LoadOpam") +def load_opam(url, opam_root, opam_instance, opam_url, opam_package): + """Load Opam's artifacts""" + return OpamLoader.from_configfile( + url=url, + opam_root=opam_root, + opam_instance=opam_instance, + opam_url=opam_url, + opam_package=opam_package, + ).load() diff --git a/swh/loader/package/opam/tests/__init__.py b/swh/loader/package/opam/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam new file mode 100644 index 0000000..d3b1def --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: "Adjustable grid (two dimensional array) library" +description: + "Adjustable grids are two dimensional arrays whose width/height can be changed by adding or removing row/column at either end (one at a time)." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/agrid" +bug-reports: "https://github.com/ocamlpro/agrid/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.7"} + "flex-array" {>= "1.2"} + "bisect_ppx" {with-test & >= "2.6" & dev} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/agrid.git" +url { + src: "https://github.com/OCamlPro/agrid/archive/0.1.tar.gz" + checksum: [ + "sha256=ea82546711a6abdd4edf8bc3052041498cae9c2e5a9e147e29820da4eac4beb4" + "sha512=f53b2c095e3607e53f92d4e7e13848e9e34bd866837335e7d9341dbb468ac46ffbcd2002d1bf1105e2f6060f57871aef7ce8e65594855447fafb72ad32b076b7" + ] +} diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam new file mode 100644 index 0000000..a533960 --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" | "os" != "win32" | "os" != "mingw"} + "ctypes" {>= "0.17.1" | "os" != "win32" | "os" != "mingw"} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.1.tar.gz" + checksum: [ + "sha256=89aa3586af3c38aea17302cfb0b6243afdf0fd90b2eb1f86fb69990917304445" + "sha512=3302f8c8c2c3ecc217f199e97d8d32d1ef14af9dd3763cbc1030522832ce375192422f7b6b6acd8cd8399d96ccd52d5e5cfffac3cb392f1cf8b9f67374c1cf7c" + ] +} diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam new file mode 100644 index 0000000..d58f79a --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" & (os = "win32" | os = "cygwin")} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.2.tar.gz" + checksum: [ + "sha256=af81e7bd7dd7125eb5168226def47e6e2287f3210ba33c015b58123a1877c40d" + "sha512=30c0db12eb453f3549bf52c202d6e6a8d12e8f4f07b92d8fc68309beb4dcf66b610eab49ac4bd3e1ddd487cd1539790c92f9511c5c06d33c2bf966148c08aa82" + ] +} diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam new file mode 100644 index 0000000..9149095 --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam @@ -0,0 +1,43 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: """ +directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. +It is inspired by similar libraries for other languages such as directories-jvm. +The following conventions are used: +- XDG Base Directory Specification and xdg-user-dirs on Linux +- Known Folders on Windows +- Standard Directories on macOS. +""" +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.3.tar.gz" + checksum: [ + "sha256=9b37d1d43e3b06f3b68ebe57651ea2cf5a9db6d5bdb1a0afe65786bcdcc8bf11" + "sha512=9d1634a0c44dd74dc3005154ebbc22c39bb0c3e1477f3e67c9fc49cc32059ac0f496816c06785f204f60ee67136c998f526ba0c9bf4232d040ff30bd89411fb3" + ] +} diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam new file mode 100644 index 0000000..64003fa --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam @@ -0,0 +1,37 @@ +opam-version: "2.0" +synopsis: "SVG badge generator" +description: + "An OCaml library for SVG badge generation. There's also a command-line tool provided." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://ocamlpro.github.io/ocb/" +doc: "https://ocamlpro.github.io/ocb/api/" +bug-reports: "https://github.com/OCamlPro/ocb/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.0"} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/OCamlPro/ocb.git" +url { + src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz" + checksum: [ + "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46" + "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b" + ] +} diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/repo b/swh/loader/package/opam/tests/data/fake_opam_repo/repo new file mode 100644 index 0000000..013b84d --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/repo @@ -0,0 +1 @@ +opam-version: "2.0" diff --git a/swh/loader/package/opam/tests/data/fake_opam_repo/version b/swh/loader/package/opam/tests/data/fake_opam_repo/version new file mode 100644 index 0000000..ac39a10 --- /dev/null +++ b/swh/loader/package/opam/tests/data/fake_opam_repo/version @@ -0,0 +1 @@ +0.9.0 diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_agrid_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_agrid_archive_0.1.tar.gz new file mode 100644 index 0000000..8bd3037 Binary files /dev/null and b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_agrid_archive_0.1.tar.gz differ diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.1.tar.gz new file mode 100644 index 0000000..3b41903 Binary files /dev/null and b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.1.tar.gz differ diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.2.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.2.tar.gz new file mode 100644 index 0000000..438991e Binary files /dev/null and b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.2.tar.gz differ diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.3.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.3.tar.gz new file mode 100644 index 0000000..6e029a6 Binary files /dev/null and b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_directories_archive_0.3.tar.gz differ diff --git a/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_ocb_archive_0.1.tar.gz b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_ocb_archive_0.1.tar.gz new file mode 100644 index 0000000..ad02c8e Binary files /dev/null and b/swh/loader/package/opam/tests/data/https_github.com/OCamlPro_ocb_archive_0.1.tar.gz differ diff --git a/swh/loader/package/opam/tests/test_opam.py b/swh/loader/package/opam/tests/test_opam.py new file mode 100644 index 0000000..9ffce58 --- /dev/null +++ b/swh/loader/package/opam/tests/test_opam.py @@ -0,0 +1,176 @@ +# Copyright (C) 2019-2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from shutil import rmtree + +from swh.loader.package.opam.loader import OpamLoader, OpamPackageInfo +from swh.loader.tests import assert_last_visit_matches, check_snapshot, get_stats +from swh.model.hashutil import hash_to_bytes +from swh.model.model import Person, Snapshot, SnapshotBranch, TargetType + + +def test_opam_loader_one_version(tmpdir, requests_mock_datadir, datadir, swh_storage): + + opam_url = f"file://{datadir}/fake_opam_repo" + + opam_root = tmpdir + # the directory should NOT exist, we just need an unique name, so we delete it + rmtree(tmpdir) + + opam_instance = "loadertest" + + opam_package = "agrid" + url = f"opam+{opam_url}/packages/{opam_package}" + + loader = OpamLoader( + swh_storage, url, opam_root, opam_instance, opam_url, opam_package + ) + + actual_load_status = loader.load() + + expected_snapshot_id = hash_to_bytes("4e4bf977312460329d7f769b0be89937c9827efc") + assert actual_load_status == { + "status": "eventful", + "snapshot_id": expected_snapshot_id.hex(), + } + + target = b"S\x8c\x8aq\xdcy\xa4/0\xa0\xb2j\xeb\xc1\x16\xad\xce\x06\xeaV" + + expected_snapshot = Snapshot( + id=expected_snapshot_id, + branches={ + b"HEAD": SnapshotBranch(target=b"agrid.0.1", target_type=TargetType.ALIAS,), + b"agrid.0.1": SnapshotBranch( + target=target, target_type=TargetType.REVISION, + ), + }, + ) + check_snapshot(expected_snapshot, swh_storage) + + assert_last_visit_matches( + swh_storage, url, status="full", type="opam", snapshot=expected_snapshot_id + ) + + stats = get_stats(swh_storage) + + assert { + "content": 18, + "directory": 8, + "origin": 1, + "origin_visit": 1, + "release": 0, + "revision": 1, + "skipped_content": 0, + "snapshot": 1, + } == stats + + +def test_opam_loader_many_version(tmpdir, requests_mock_datadir, datadir, swh_storage): + + opam_url = f"file://{datadir}/fake_opam_repo" + + opam_root = tmpdir + # the directory should NOT exist, we just need an unique name, so we delete it + rmtree(tmpdir) + + opam_instance = "loadertest" + + opam_package = "directories" + url = f"opam+{opam_url}/packages/{opam_package}" + + loader = OpamLoader( + swh_storage, url, opam_root, opam_instance, opam_url, opam_package + ) + + actual_load_status = loader.load() + + expected_snapshot_id = hash_to_bytes("1b49be175dcf17c0f568bcd7aac3d4faadc41249") + assert actual_load_status == { + "status": "eventful", + "snapshot_id": expected_snapshot_id.hex(), + } + + expected_snapshot = Snapshot( + id=expected_snapshot_id, + branches={ + b"HEAD": SnapshotBranch( + target=b"directories.0.3", target_type=TargetType.ALIAS, + ), + b"directories.0.1": SnapshotBranch( + target=b"N\x92jA\xb2\x892\xeb\xcc\x9c\xa9\xb3\xea\xa7kz\xb08\xa6V", + target_type=TargetType.REVISION, + ), + b"directories.0.2": SnapshotBranch( + target=b"yj\xc9\x1a\x8f\xe0\xaa\xff[\x88\xffz" + b"\x91C\xcc\x96\xb7\xd4\xf65", + target_type=TargetType.REVISION, + ), + b"directories.0.3": SnapshotBranch( + target=b"hA \xc4\xb5\x18A8\xb8C\x12\xa3\xa5T\xb7/v\x85X\xcb", + target_type=TargetType.REVISION, + ), + }, + ) + + check_snapshot(expected_snapshot, swh_storage) + + assert_last_visit_matches( + swh_storage, url, status="full", type="opam", snapshot=expected_snapshot_id + ) + + +def test_opam_revision(tmpdir, requests_mock_datadir, swh_storage, datadir): + + opam_url = f"file://{datadir}/fake_opam_repo" + + opam_root = tmpdir + # the directory should NOT exist, we just need an unique name, so we delete it + rmtree(tmpdir) + + opam_instance = "loadertest" + + opam_package = "ocb" + url = f"opam+{opam_url}/packages/{opam_package}" + + loader = OpamLoader( + swh_storage, url, opam_root, opam_instance, opam_url, opam_package + ) + + actual_load_status = loader.load() + + expected_snapshot_id = hash_to_bytes("398df115b9feb2f463efd21941d69b7d59cd9025") + assert actual_load_status == { + "status": "eventful", + "snapshot_id": expected_snapshot_id.hex(), + } + + info_iter = loader.get_package_info("0.1") + branch_name, package_info = next(info_iter) + expected_branch_name = "ocb.0.1" + expected_package_info = OpamPackageInfo( + url="https://github.com/OCamlPro/ocb/archive/0.1.tar.gz", + filename=None, + directory_extrinsic_metadata=[], + author=Person( + fullname=b"OCamlPro ", name=None, email=None + ), + committer=Person( + fullname=b"OCamlPro ", name=None, email=None + ), + version="0.1", + ) + + assert branch_name == expected_branch_name + assert package_info == expected_package_info + + revision_id = b"o\xad\x7f=\x07\xbb\xaah\xdbI(\xb0'\x10z\xfc\xff\x06x\x1b" + + revision = swh_storage.revision_get([revision_id])[0] + + assert revision is not None + + assert revision.author == expected_package_info.author + assert revision.committer == expected_package_info.committer diff --git a/swh/loader/package/opam/tests/test_tasks.py b/swh/loader/package/opam/tests/test_tasks.py new file mode 100644 index 0000000..8fe996c --- /dev/null +++ b/swh/loader/package/opam/tests/test_tasks.py @@ -0,0 +1,27 @@ +# Copyright (C) 2019-2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_tasks_opam_loader( + mocker, swh_scheduler_celery_app, swh_scheduler_celery_worker, swh_config +): + mock_load = mocker.patch("swh.loader.package.opam.loader.OpamLoader.load") + mock_load.return_value = {"status": "eventful"} + + res = swh_scheduler_celery_app.send_task( + "swh.loader.package.opam.tasks.LoadOpam", + args=( + "opam+https://opam.ocaml.org/packages/agrid", # url + "/tmp/test_tasks_opam_loader", # opam_root + "test_tasks_opam_loader", # opam_instance + "https://opam.ocaml.org", # opam_url + "agrid", # opam_package + ), + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}