diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 8dfc445..e7f5577 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,7 +1,8 @@ Archit Agrawal Avi Kelman (fiendish) Léni Gauffier Yann Gautier Sushant Sushant Hezekiah Maina Boris Baldassari +Léo Andrès diff --git a/setup.py b/setup.py index c8a6898..78aa42d 100755 --- a/setup.py +++ b/setup.py @@ -1,87 +1,88 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.lister", description="Software Heritage lister", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DLSGH/", packages=find_packages(), install_requires=parse_requirements() + parse_requirements("swh"), tests_require=parse_requirements("test"), setup_requires=["setuptools-scm"], extras_require={"testing": parse_requirements("test")}, use_scm_version=True, include_package_data=True, entry_points=""" [swh.cli.subcommands] lister=swh.lister.cli [swh.workers] lister.bitbucket=swh.lister.bitbucket:register lister.cgit=swh.lister.cgit:register lister.cran=swh.lister.cran:register lister.debian=swh.lister.debian:register lister.gitea=swh.lister.gitea:register lister.github=swh.lister.github:register lister.gitlab=swh.lister.gitlab:register lister.gnu=swh.lister.gnu:register lister.launchpad=swh.lister.launchpad:register lister.npm=swh.lister.npm:register + lister.opam=swh.lister.opam:register lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register lister.sourceforge=swh.lister.sourceforge:register lister.tuleap=swh.lister.tuleap:register """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-lister", "Documentation": "https://docs.softwareheritage.org/devel/swh-lister/", }, ) diff --git a/swh/lister/opam/__init__.py b/swh/lister/opam/__init__.py new file mode 100644 index 0000000..a8342cf --- /dev/null +++ b/swh/lister/opam/__init__.py @@ -0,0 +1,12 @@ +# Copyright (C) 2021 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def register(): + from .lister import OpamLister + + return { + "lister": OpamLister, + "task_modules": ["%s.tasks" % __name__], + } diff --git a/swh/lister/opam/lister.py b/swh/lister/opam/lister.py new file mode 100644 index 0000000..098e4be --- /dev/null +++ b/swh/lister/opam/lister.py @@ -0,0 +1,95 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import io +import logging +import os +from subprocess import PIPE, Popen, call +import tempfile +from typing import Iterator + +from swh.lister.pattern import StatelessLister +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin + +from ..pattern import CredentialsType + +logger = logging.getLogger(__name__) + +PageType = str + + +class OpamLister(StatelessLister[PageType]): + """ + List all repositories hosted on an opam repository. + + On initialisation, we create an opam root, with no ocaml compiler (no switch) + as we won't need it and it's costly. In this opam root, we add a single opam + repository (url) and give it a name (instance). Then, to get pages, we just ask + opam to list all the packages for our opam repository in our opam root. + + Args: + url: base URL of an opam repository + (for instance https://opam.ocaml.org) + instance: string identifier for the listed repository + + """ + + # Part of the lister API, that identifies this lister + LISTER_NAME = "opam" + + def __init__( + self, + scheduler: SchedulerInterface, + url: str, + instance: str, + credentials: CredentialsType = None, + ): + super().__init__( + scheduler=scheduler, credentials=credentials, url=url, instance=instance, + ) + self.env = os.environ.copy() + self.env["OPAMROOT"] = tempfile.mkdtemp(prefix="swh_opam_lister") + call( + ["opam", "init", "--reinit", "--bare", "--no-setup", instance, url], + env=self.env, + ) + + def get_pages(self) -> Iterator[PageType]: + proc = Popen( + [ + "opam", + "list", + "--all", + "--no-switch", + "--repos", + self.instance, + "--normalise", + "--short", + ], + env=self.env, + stdout=PIPE, + ) + if proc.stdout is not None: + for line in io.TextIOWrapper(proc.stdout): + yield line.rstrip("\n") + + def get_origins_from_page(self, page: PageType) -> Iterator[ListedOrigin]: + """Convert a page of OpamLister repositories into a list of ListedOrigins""" + assert self.lister_obj.id is not None + # a page is just a package name + url = f"opam+{self.url}/packages/{page}/" + yield ListedOrigin( + lister_id=self.lister_obj.id, + visit_type="opam", + url=url, + last_update=None, + extra_loader_arguments={ + "opam_root": self.env["OPAMROOT"], + "opam_instance": self.instance, + "opam_url": self.url, + "opam_package": page, + }, + ) diff --git a/swh/lister/opam/tasks.py b/swh/lister/opam/tasks.py new file mode 100644 index 0000000..9dfe3be --- /dev/null +++ b/swh/lister/opam/tasks.py @@ -0,0 +1,18 @@ +# Copyright (C) 2021 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.lister.opam.lister import OpamLister + + +@shared_task(name=__name__ + ".OpamListerTask") +def list_opam(**lister_args): + """Lister task for the Opam registry""" + return OpamLister.from_configfile(**lister_args).run().dict() + + +@shared_task(name=__name__ + ".ping") +def _ping(): + return "OK" diff --git a/swh/lister/opam/tests/__init__.py b/swh/lister/opam/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam new file mode 100644 index 0000000..d3b1def --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: "Adjustable grid (two dimensional array) library" +description: + "Adjustable grids are two dimensional arrays whose width/height can be changed by adding or removing row/column at either end (one at a time)." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/agrid" +bug-reports: "https://github.com/ocamlpro/agrid/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.7"} + "flex-array" {>= "1.2"} + "bisect_ppx" {with-test & >= "2.6" & dev} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/agrid.git" +url { + src: "https://github.com/OCamlPro/agrid/archive/0.1.tar.gz" + checksum: [ + "sha256=ea82546711a6abdd4edf8bc3052041498cae9c2e5a9e147e29820da4eac4beb4" + "sha512=f53b2c095e3607e53f92d4e7e13848e9e34bd866837335e7d9341dbb468ac46ffbcd2002d1bf1105e2f6060f57871aef7ce8e65594855447fafb72ad32b076b7" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam new file mode 100644 index 0000000..c01e684 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam @@ -0,0 +1,57 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+http://github.com/c-cube/calculon.git" +build: [ + [ + "./configure" + "--bindir" + "%{bin}%" + "--%{uri+lambdasoup+cohttp+atdgen:enable}%-web" + "--%{re+sequence:enable}%-extras" + ] + [make "build"] + [make "test"] {with-test} + [make "doc"] {with-doc} +] +install: [make "install"] +remove: ["ocamlfind" "remove" "calculon"] +depends: [ + "ocaml" {>= "4.02.0"} + "ocamlfind" {build} + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.4.0" & < "0.6.0"} + "tls" + "yojson" + "containers" {>= "1.0" & < "2.0"} + "ISO8601" + "re" {>= "1.5.0" & < "1.7.2"} + "stringext" +] +depopts: [ + "uri" + "cohttp" + "atdgen" + "lambdasoup" + "sequence" +] +conflicts: [ + "cohttp" {>= "1.0.0"} +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +flags: light-uninstall +url { + src: "https://github.com/c-cube/calculon/archive/0.1.tar.gz" + checksum: "md5=24fb37bb915717a0497962fda9fecc5c" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam new file mode 100644 index 0000000..4da90dd --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam @@ -0,0 +1,46 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+https://github.com/c-cube/calculon.git" +build: [ + ["jbuilder" "build" "-p" name] + ["jbuilder" "runtest" "-p" name] {with-test} + ["jbuilder" "build" "@doc" "-p" name] {with-doc} +] +depends: [ + "ocaml" {>= "4.02.0"} + "jbuilder" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.4.0" & < "0.6.0"} + "tls" + "yojson" + "containers" {>= "1.0" & < "2.1"} + "ISO8601" + "stringext" + "re" +] +depopts: [ + "uri" + "cohttp" + "cohttp-lwt" + "cohttp-lwt-unix" + "atdgen" + "lambdasoup" + "sequence" +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +url { + src: "https://github.com/c-cube/calculon/archive/0.2.tar.gz" + checksum: "md5=df6692d28d27ae6f87c773739b758fb8" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam new file mode 100644 index 0000000..47c0849 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam @@ -0,0 +1,40 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+https://github.com/c-cube/calculon.git" +build: [ + ["jbuilder" "build" "-p" name] + ["jbuilder" "runtest" "-p" name] {with-test} + ["jbuilder" "build" "@doc" "-p" name] {with-doc} +] +depends: [ + "ocaml" {>= "4.02.0"} + "jbuilder" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.6.0"} + "irc-client-lwt" + "irc-client-tls" + "tls" + "yojson" + "containers" {>= "1.0" & < "3.0"} + "ISO8601" + "stringext" + "re" {>= "1.7.2"} + "odoc" {with-doc} +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +url { + src: "https://github.com/c-cube/calculon/archive/0.3.tar.gz" + checksum: "md5=635a554938d42319beeb6d70a85800ff" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam new file mode 100644 index 0000000..f0fa919 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam @@ -0,0 +1,40 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +tags: ["irc" "bot" "factoids"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +depends: [ + "dune" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.6.0"} + "irc-client-lwt" + "irc-client-tls" + "tls" + "yojson" + "containers" {>= "1.0" & < "3.0"} + "ISO8601" + "stringext" + "x509" {< "0.10.0"} + "re" {>= "1.7.2"} + "odoc" {with-doc} + "ocaml" {>= "4.03.0"} + "mdx" {with-test} +] +build: [ + ["dune" "build" "-p" name "-j" jobs] + ["dune" "build" "@doc" "-p" name] {with-doc} + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.4.tar.gz" + checksum: [ + "md5=e53363833b7e3620a2ce7ffb9a27715d" + "sha512=c997fd52fd277e8a2d50f266f12182afa96f72bb2f161a6ac1b3372fc9ebbcc12c7593f90cbb5873456b8b96d3488e5c52d071723bbaa05097710d643241d70b" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam new file mode 100644 index 0000000..e9db0a4 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +authors: ["Armael" "Enjolras" "c-cube"] +maintainer: "c-cube" +build: [ + ["dune" "build" "-p" name "-j" jobs] + [ "dune" "build" "@doc" "-p" name] {with-doc} + [ "dune" "runtest" "-p" name "-j" jobs] {with-test} +] +depends: [ + "dune" { >= "1.1" } + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" { >= "0.6.0" } + "irc-client-lwt" + "irc-client-lwt-ssl" + "yojson" + "containers" { >= "1.0" & < "3.0" } + "ISO8601" + "stringext" + "re" { >= "1.7.2" } + "iter" + "odoc" {with-doc} + "ocaml" { >= "4.03.0" } +] +tags: [ "irc" "bot" "factoids" ] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.5.tar.gz" + checksum: [ + "md5=831b8d45ac76bfa1118e7e954bfd4474" + "sha512=b7e856d88a2c34f2f7bb2c5c8f416ef99e29ccd46a9016e5f7fefc838df6fcb5daffd45170b606562a2ba15e910421884071e6e19fa90b23f412f45d85cc7d5a" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam new file mode 100644 index 0000000..3a9910d --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam @@ -0,0 +1,39 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +authors: ["Armael" "Enjolras" "c-cube"] +maintainer: "c-cube" +build: [ + ["dune" "build" "-p" name "-j" jobs] + ["dune" "build" "@doc" "-p" name] {with-doc} + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +depends: [ + "dune" { >= "1.1" } + "base-unix" + "lwt" + "irc-client" { >= "0.6.0" } + "irc-client-lwt" + "irc-client-lwt-ssl" + "logs" + "yojson" { >= "1.6" } + "containers" { >= "1.0" & < "3.0" } + "ISO8601" + "stringext" + "re" { >= "1.7.2" } + "odoc" {with-doc} + "ocaml" { >= "4.03.0" } +] +depopts: [ + "iter" +] +tags: [ "irc" "bot" "factoids" ] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.6.tar.gz" + checksum: [ + "md5=16ad257c16f82a2acfab5e10c7c8ef8a" + "sha512=c460994c0ffabf0d756cdbb0cdd77b99d3b4844f597c894bb2c8ed22d1038b44f1be4d76721a956a76cc953d915fb76324f0fdb385e6a2e531da7fd4cc832836" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam new file mode 100644 index 0000000..a533960 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" | "os" != "win32" | "os" != "mingw"} + "ctypes" {>= "0.17.1" | "os" != "win32" | "os" != "mingw"} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.1.tar.gz" + checksum: [ + "sha256=89aa3586af3c38aea17302cfb0b6243afdf0fd90b2eb1f86fb69990917304445" + "sha512=3302f8c8c2c3ecc217f199e97d8d32d1ef14af9dd3763cbc1030522832ce375192422f7b6b6acd8cd8399d96ccd52d5e5cfffac3cb392f1cf8b9f67374c1cf7c" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam new file mode 100644 index 0000000..d58f79a --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" & (os = "win32" | os = "cygwin")} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.2.tar.gz" + checksum: [ + "sha256=af81e7bd7dd7125eb5168226def47e6e2287f3210ba33c015b58123a1877c40d" + "sha512=30c0db12eb453f3549bf52c202d6e6a8d12e8f4f07b92d8fc68309beb4dcf66b610eab49ac4bd3e1ddd487cd1539790c92f9511c5c06d33c2bf966148c08aa82" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam new file mode 100644 index 0000000..9149095 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam @@ -0,0 +1,43 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: """ +directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. +It is inspired by similar libraries for other languages such as directories-jvm. +The following conventions are used: +- XDG Base Directory Specification and xdg-user-dirs on Linux +- Known Folders on Windows +- Standard Directories on macOS. +""" +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.3.tar.gz" + checksum: [ + "sha256=9b37d1d43e3b06f3b68ebe57651ea2cf5a9db6d5bdb1a0afe65786bcdcc8bf11" + "sha512=9d1634a0c44dd74dc3005154ebbc22c39bb0c3e1477f3e67c9fc49cc32059ac0f496816c06785f204f60ee67136c998f526ba0c9bf4232d040ff30bd89411fb3" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam new file mode 100644 index 0000000..64003fa --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam @@ -0,0 +1,37 @@ +opam-version: "2.0" +synopsis: "SVG badge generator" +description: + "An OCaml library for SVG badge generation. There's also a command-line tool provided." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://ocamlpro.github.io/ocb/" +doc: "https://ocamlpro.github.io/ocb/api/" +bug-reports: "https://github.com/OCamlPro/ocb/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.0"} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/OCamlPro/ocb.git" +url { + src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz" + checksum: [ + "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46" + "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/repo b/swh/lister/opam/tests/data/fake_opam_repo/repo new file mode 100644 index 0000000..013b84d --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/repo @@ -0,0 +1 @@ +opam-version: "2.0" diff --git a/swh/lister/opam/tests/data/fake_opam_repo/version b/swh/lister/opam/tests/data/fake_opam_repo/version new file mode 100644 index 0000000..ac39a10 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/version @@ -0,0 +1 @@ +0.9.0 diff --git a/swh/lister/opam/tests/test_lister.py b/swh/lister/opam/tests/test_lister.py new file mode 100644 index 0000000..763f491 --- /dev/null +++ b/swh/lister/opam/tests/test_lister.py @@ -0,0 +1,65 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import io +from unittest.mock import MagicMock + +from swh.lister.opam.lister import OpamLister + + +def test_urls(swh_scheduler, mocker): + + instance_url = "https://opam.ocaml.org" + + lister = OpamLister(swh_scheduler, url=instance_url, instance="opam") + + mocked_popen = MagicMock() + mocked_popen.stdout = io.BytesIO(b"bar\nbaz\nfoo\n") + + # replaces the real Popen with a fake one + mocker.patch("swh.lister.opam.lister.Popen", return_value=mocked_popen) + + # call the lister and get all listed origins urls + stats = lister.run() + + assert stats.pages == 3 + assert stats.origins == 3 + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + expected_urls = [ + f"opam+{instance_url}/packages/bar/", + f"opam+{instance_url}/packages/baz/", + f"opam+{instance_url}/packages/foo/", + ] + + result_urls = [origin.url for origin in scheduler_origins] + + assert expected_urls == result_urls + + +def test_opam_binary(datadir, swh_scheduler): + + instance_url = f"file://{datadir}/fake_opam_repo" + + lister = OpamLister(swh_scheduler, url=instance_url, instance="fake") + + stats = lister.run() + + assert stats.pages == 4 + assert stats.origins == 4 + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + expected_urls = [ + f"opam+{instance_url}/packages/agrid/", + f"opam+{instance_url}/packages/calculon/", + f"opam+{instance_url}/packages/directories/", + f"opam+{instance_url}/packages/ocb/", + ] + + result_urls = [origin.url for origin in scheduler_origins] + + assert expected_urls == result_urls diff --git a/swh/lister/opam/tests/test_tasks.py b/swh/lister/opam/tests/test_tasks.py new file mode 100644 index 0000000..5b04bf1 --- /dev/null +++ b/swh/lister/opam/tests/test_tasks.py @@ -0,0 +1,31 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.lister.pattern import ListerStats + + +def test_opam_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.ping") + assert res + res.wait() + assert res.successful() + assert res.result == "OK" + + +def test_opam_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker): + # setup the mocked OpamLister + lister = mocker.patch("swh.lister.opam.tasks.OpamLister") + lister.from_configfile.return_value = lister + stats = ListerStats(pages=42, origins=42) + lister.run.return_value = stats + + res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.OpamListerTask") + assert res + res.wait() + assert res.successful() + assert res.result == stats.dict() + + lister.from_configfile.assert_called_once_with() + lister.run.assert_called_once_with() diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py index 4a0bff3..4b8e7ae 100644 --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -1,43 +1,44 @@ # Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.lister.cli import SUPPORTED_LISTERS, get_lister lister_args = { "cgit": {"url": "https://git.eclipse.org/c/",}, "phabricator": { "instance": "softwareheritage", "url": "https://forge.softwareheritage.org/api/diffusion.repository.search", "api_token": "bogus", }, "gitea": {"url": "https://try.gitea.io/api/v1/",}, "tuleap": {"url": "https://tuleap.net",}, "gitlab": {"url": "https://gitlab.ow2.org/api/v4", "instance": "ow2",}, + "opam": {"url": "https://opam.ocaml.org", "instance": "opam"}, } def test_get_lister_wrong_input(): """Unsupported lister should raise""" with pytest.raises(ValueError) as e: get_lister("unknown", "db-url") assert "Invalid lister" in str(e.value) def test_get_lister(swh_scheduler_config): """Instantiating a supported lister should be ok """ # Drop launchpad lister from the lister to check, its test setup is more involved # than the other listers and it's not currently done here for lister_name in SUPPORTED_LISTERS: lst = get_lister( lister_name, scheduler={"cls": "local", **swh_scheduler_config}, **lister_args.get(lister_name, {}), ) assert hasattr(lst, "run")