diff --git a/CONTRIBUTORS b/CONTRIBUTORS --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -5,3 +5,4 @@ Sushant Sushant Hezekiah Maina Boris Baldassari +Léo Andrès diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ lister.gnu=swh.lister.gnu:register lister.launchpad=swh.lister.launchpad:register lister.npm=swh.lister.npm:register + lister.opam=swh.lister.opam:register lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register diff --git a/swh/lister/opam/__init__.py b/swh/lister/opam/__init__.py new file mode 100644 --- /dev/null +++ b/swh/lister/opam/__init__.py @@ -0,0 +1,12 @@ +# Copyright (C) 2021 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def register(): + from .lister import OpamLister + + return { + "lister": OpamLister, + "task_modules": ["%s.tasks" % __name__], + } diff --git a/swh/lister/opam/lister.py b/swh/lister/opam/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/opam/lister.py @@ -0,0 +1,95 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import io +import logging +import os +from subprocess import PIPE, Popen, call +import tempfile +from typing import Iterator + +from swh.lister.pattern import StatelessLister +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin + +from ..pattern import CredentialsType + +logger = logging.getLogger(__name__) + +PageType = str + + +class OpamLister(StatelessLister[PageType]): + """ + List all repositories hosted on an opam repository. + + On initialisation, we create an opam root, with no ocaml compiler (no switch) + as we won't need it and it's costly. In this opam root, we add a single opam + repository (url) and give it a name (instance). Then, to get pages, we just ask + opam to list all the packages for our opam repository in our opam root. + + Args: + url: base URL of an opam repository + (for instance https://opam.ocaml.org) + instance: string identifier for the listed repository + + """ + + # Part of the lister API, that identifies this lister + LISTER_NAME = "opam" + + def __init__( + self, + scheduler: SchedulerInterface, + url: str, + instance: str, + credentials: CredentialsType = None, + ): + super().__init__( + scheduler=scheduler, credentials=credentials, url=url, instance=instance, + ) + self.env = os.environ.copy() + self.env["OPAMROOT"] = tempfile.mkdtemp(prefix="swh_opam_lister") + call( + ["opam", "init", "--reinit", "--bare", "--no-setup", instance, url], + env=self.env, + ) + + def get_pages(self) -> Iterator[PageType]: + proc = Popen( + [ + "opam", + "list", + "--all", + "--no-switch", + "--repos", + self.instance, + "--normalise", + "--short", + ], + env=self.env, + stdout=PIPE, + ) + if proc.stdout is not None: + for line in io.TextIOWrapper(proc.stdout): + yield line.rstrip("\n") + + def get_origins_from_page(self, page: PageType) -> Iterator[ListedOrigin]: + """Convert a page of OpamLister repositories into a list of ListedOrigins""" + assert self.lister_obj.id is not None + # a page is just a package name + url = f"opam+{self.url}/packages/{page}/" + yield ListedOrigin( + lister_id=self.lister_obj.id, + visit_type="opam", + url=url, + last_update=None, + extra_loader_arguments={ + "opam_root": self.env["OPAMROOT"], + "opam_instance": self.instance, + "opam_url": self.url, + "opam_package": page, + }, + ) diff --git a/swh/lister/opam/tasks.py b/swh/lister/opam/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tasks.py @@ -0,0 +1,18 @@ +# Copyright (C) 2021 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.lister.opam.lister import OpamLister + + +@shared_task(name=__name__ + ".OpamListerTask") +def list_opam(**lister_args): + """Lister task for the Opam registry""" + return OpamLister.from_configfile(**lister_args).run().dict() + + +@shared_task(name=__name__ + ".ping") +def _ping(): + return "OK" diff --git a/swh/lister/opam/tests/__init__.py b/swh/lister/opam/tests/__init__.py new file mode 100644 diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/agrid/agrid.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: "Adjustable grid (two dimensional array) library" +description: + "Adjustable grids are two dimensional arrays whose width/height can be changed by adding or removing row/column at either end (one at a time)." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/agrid" +bug-reports: "https://github.com/ocamlpro/agrid/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.7"} + "flex-array" {>= "1.2"} + "bisect_ppx" {with-test & >= "2.6" & dev} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/agrid.git" +url { + src: "https://github.com/OCamlPro/agrid/archive/0.1.tar.gz" + checksum: [ + "sha256=ea82546711a6abdd4edf8bc3052041498cae9c2e5a9e147e29820da4eac4beb4" + "sha512=f53b2c095e3607e53f92d4e7e13848e9e34bd866837335e7d9341dbb468ac46ffbcd2002d1bf1105e2f6060f57871aef7ce8e65594855447fafb72ad32b076b7" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.1/opam @@ -0,0 +1,57 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+http://github.com/c-cube/calculon.git" +build: [ + [ + "./configure" + "--bindir" + "%{bin}%" + "--%{uri+lambdasoup+cohttp+atdgen:enable}%-web" + "--%{re+sequence:enable}%-extras" + ] + [make "build"] + [make "test"] {with-test} + [make "doc"] {with-doc} +] +install: [make "install"] +remove: ["ocamlfind" "remove" "calculon"] +depends: [ + "ocaml" {>= "4.02.0"} + "ocamlfind" {build} + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.4.0" & < "0.6.0"} + "tls" + "yojson" + "containers" {>= "1.0" & < "2.0"} + "ISO8601" + "re" {>= "1.5.0" & < "1.7.2"} + "stringext" +] +depopts: [ + "uri" + "cohttp" + "atdgen" + "lambdasoup" + "sequence" +] +conflicts: [ + "cohttp" {>= "1.0.0"} +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +flags: light-uninstall +url { + src: "https://github.com/c-cube/calculon/archive/0.1.tar.gz" + checksum: "md5=24fb37bb915717a0497962fda9fecc5c" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.2/opam @@ -0,0 +1,46 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+https://github.com/c-cube/calculon.git" +build: [ + ["jbuilder" "build" "-p" name] + ["jbuilder" "runtest" "-p" name] {with-test} + ["jbuilder" "build" "@doc" "-p" name] {with-doc} +] +depends: [ + "ocaml" {>= "4.02.0"} + "jbuilder" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.4.0" & < "0.6.0"} + "tls" + "yojson" + "containers" {>= "1.0" & < "2.1"} + "ISO8601" + "stringext" + "re" +] +depopts: [ + "uri" + "cohttp" + "cohttp-lwt" + "cohttp-lwt-unix" + "atdgen" + "lambdasoup" + "sequence" +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +url { + src: "https://github.com/c-cube/calculon/archive/0.2.tar.gz" + checksum: "md5=df6692d28d27ae6f87c773739b758fb8" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.3/opam @@ -0,0 +1,40 @@ +opam-version: "2.0" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +tags: ["irc" "bot" "factoids"] +dev-repo: "git+https://github.com/c-cube/calculon.git" +build: [ + ["jbuilder" "build" "-p" name] + ["jbuilder" "runtest" "-p" name] {with-test} + ["jbuilder" "build" "@doc" "-p" name] {with-doc} +] +depends: [ + "ocaml" {>= "4.02.0"} + "jbuilder" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.6.0"} + "irc-client-lwt" + "irc-client-tls" + "tls" + "yojson" + "containers" {>= "1.0" & < "3.0"} + "ISO8601" + "stringext" + "re" {>= "1.7.2"} + "odoc" {with-doc} +] +synopsis: + "Library for writing IRC bots in OCaml, a collection of plugins, and a dramatic robotic actor." +description: """ +The core library is called `calculon` and revolves around +the concept of commands that react to user messages. See the README for a small +tutorial on how to write your own plugins.""" +url { + src: "https://github.com/c-cube/calculon/archive/0.3.tar.gz" + checksum: "md5=635a554938d42319beeb6d70a85800ff" +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.4/opam @@ -0,0 +1,40 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +maintainer: "c-cube" +authors: ["Armael" "Enjolras" "c-cube"] +tags: ["irc" "bot" "factoids"] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +depends: [ + "dune" + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" {>= "0.6.0"} + "irc-client-lwt" + "irc-client-tls" + "tls" + "yojson" + "containers" {>= "1.0" & < "3.0"} + "ISO8601" + "stringext" + "x509" {< "0.10.0"} + "re" {>= "1.7.2"} + "odoc" {with-doc} + "ocaml" {>= "4.03.0"} + "mdx" {with-test} +] +build: [ + ["dune" "build" "-p" name "-j" jobs] + ["dune" "build" "@doc" "-p" name] {with-doc} + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.4.tar.gz" + checksum: [ + "md5=e53363833b7e3620a2ce7ffb9a27715d" + "sha512=c997fd52fd277e8a2d50f266f12182afa96f72bb2f161a6ac1b3372fc9ebbcc12c7593f90cbb5873456b8b96d3488e5c52d071723bbaa05097710d643241d70b" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.5/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +authors: ["Armael" "Enjolras" "c-cube"] +maintainer: "c-cube" +build: [ + ["dune" "build" "-p" name "-j" jobs] + [ "dune" "build" "@doc" "-p" name] {with-doc} + [ "dune" "runtest" "-p" name "-j" jobs] {with-test} +] +depends: [ + "dune" { >= "1.1" } + "base-bytes" + "base-unix" + "result" + "lwt" + "irc-client" { >= "0.6.0" } + "irc-client-lwt" + "irc-client-lwt-ssl" + "yojson" + "containers" { >= "1.0" & < "3.0" } + "ISO8601" + "stringext" + "re" { >= "1.7.2" } + "iter" + "odoc" {with-doc} + "ocaml" { >= "4.03.0" } +] +tags: [ "irc" "bot" "factoids" ] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.5.tar.gz" + checksum: [ + "md5=831b8d45ac76bfa1118e7e954bfd4474" + "sha512=b7e856d88a2c34f2f7bb2c5c8f416ef99e29ccd46a9016e5f7fefc838df6fcb5daffd45170b606562a2ba15e910421884071e6e19fa90b23f412f45d85cc7d5a" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/calculon/calculon.0.6/opam @@ -0,0 +1,39 @@ +opam-version: "2.0" +synopsis: "Library for writing IRC bots in OCaml and a collection of plugins" +authors: ["Armael" "Enjolras" "c-cube"] +maintainer: "c-cube" +build: [ + ["dune" "build" "-p" name "-j" jobs] + ["dune" "build" "@doc" "-p" name] {with-doc} + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +depends: [ + "dune" { >= "1.1" } + "base-unix" + "lwt" + "irc-client" { >= "0.6.0" } + "irc-client-lwt" + "irc-client-lwt-ssl" + "logs" + "yojson" { >= "1.6" } + "containers" { >= "1.0" & < "3.0" } + "ISO8601" + "stringext" + "re" { >= "1.7.2" } + "odoc" {with-doc} + "ocaml" { >= "4.03.0" } +] +depopts: [ + "iter" +] +tags: [ "irc" "bot" "factoids" ] +homepage: "https://github.com/c-cube/calculon" +bug-reports: "https://github.com/c-cube/calculon/issues" +dev-repo: "git+https://github.com/c-cube/calculon.git" +url { + src: "https://github.com/c-cube/calculon/archive/v0.6.tar.gz" + checksum: [ + "md5=16ad257c16f82a2acfab5e10c7c8ef8a" + "sha512=c460994c0ffabf0d756cdbb0cdd77b99d3b4844f597c894bb2c8ed22d1038b44f1be4d76721a956a76cc953d915fb76324f0fdb385e6a2e531da7fd4cc832836" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.1/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" | "os" != "win32" | "os" != "mingw"} + "ctypes" {>= "0.17.1" | "os" != "win32" | "os" != "mingw"} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.1.tar.gz" + checksum: [ + "sha256=89aa3586af3c38aea17302cfb0b6243afdf0fd90b2eb1f86fb69990917304445" + "sha512=3302f8c8c2c3ecc217f199e97d8d32d1ef14af9dd3763cbc1030522832ce375192422f7b6b6acd8cd8399d96ccd52d5e5cfffac3cb392f1cf8b9f67374c1cf7c" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.2/opam @@ -0,0 +1,38 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: + "directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. It is inspired by similar libraries for other languages such as directories-jvm. The following conventions are used: XDG Base Directory Specification and xdg-user-dirs on Linux, Known Folders on Windows, Standard Directories on macOS." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes-foreign" {>= "0.4.0" & (os = "win32" | os = "cygwin")} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.2.tar.gz" + checksum: [ + "sha256=af81e7bd7dd7125eb5168226def47e6e2287f3210ba33c015b58123a1877c40d" + "sha512=30c0db12eb453f3549bf52c202d6e6a8d12e8f4f07b92d8fc68309beb4dcf66b610eab49ac4bd3e1ddd487cd1539790c92f9511c5c06d33c2bf966148c08aa82" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/directories/directories.0.3/opam @@ -0,0 +1,43 @@ +opam-version: "2.0" +synopsis: + "An OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows" +description: """ +directories is an OCaml library that provides configuration, cache and data paths (and more!) following the suitable conventions on Linux, macOS and Windows. +It is inspired by similar libraries for other languages such as directories-jvm. +The following conventions are used: +- XDG Base Directory Specification and xdg-user-dirs on Linux +- Known Folders on Windows +- Standard Directories on macOS. +""" +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://github.com/ocamlpro/directories" +bug-reports: "https://github.com/ocamlpro/directories/issues" +depends: [ + "dune" {>= "2.1"} + "ocaml" {>= "4.07.0"} + "ctypes" {>= "0.17.1" & (os = "win32" | os = "cygwin")} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/ocamlpro/directories.git" +url { + src: "https://github.com/OCamlPro/directories/archive/0.3.tar.gz" + checksum: [ + "sha256=9b37d1d43e3b06f3b68ebe57651ea2cf5a9db6d5bdb1a0afe65786bcdcc8bf11" + "sha512=9d1634a0c44dd74dc3005154ebbc22c39bb0c3e1477f3e67c9fc49cc32059ac0f496816c06785f204f60ee67136c998f526ba0c9bf4232d040ff30bd89411fb3" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam b/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/packages/ocb/ocb.0.1/opam @@ -0,0 +1,37 @@ +opam-version: "2.0" +synopsis: "SVG badge generator" +description: + "An OCaml library for SVG badge generation. There's also a command-line tool provided." +maintainer: ["OCamlPro "] +authors: ["OCamlPro "] +license: "ISC" +homepage: "https://ocamlpro.github.io/ocb/" +doc: "https://ocamlpro.github.io/ocb/api/" +bug-reports: "https://github.com/OCamlPro/ocb/issues" +depends: [ + "ocaml" {>= "4.05"} + "dune" {>= "2.0"} + "odoc" {with-doc} +] +build: [ + ["dune" "subst"] {dev} + [ + "dune" + "build" + "-p" + name + "-j" + jobs + "@install" + "@runtest" {with-test} + "@doc" {with-doc} + ] +] +dev-repo: "git+https://github.com/OCamlPro/ocb.git" +url { + src: "https://github.com/OCamlPro/ocb/archive/0.1.tar.gz" + checksum: [ + "sha256=aa27684fbda1b8036ae7e3c87de33a98a9cd2662bcc91c8447e00e41476b6a46" + "sha512=1260344f184dd8c8074b0439dbcc8a5d59550a654c249cd61913d4c150c664f37b76195ddca38f7f6646d08bddb320ceb8d420508450b4f09a233cd5c22e6b9b" + ] +} diff --git a/swh/lister/opam/tests/data/fake_opam_repo/repo b/swh/lister/opam/tests/data/fake_opam_repo/repo new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/repo @@ -0,0 +1 @@ +opam-version: "2.0" diff --git a/swh/lister/opam/tests/data/fake_opam_repo/version b/swh/lister/opam/tests/data/fake_opam_repo/version new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/data/fake_opam_repo/version @@ -0,0 +1 @@ +0.9.0 diff --git a/swh/lister/opam/tests/test_lister.py b/swh/lister/opam/tests/test_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/test_lister.py @@ -0,0 +1,65 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import io +from unittest.mock import MagicMock + +from swh.lister.opam.lister import OpamLister + + +def test_urls(swh_scheduler, mocker): + + instance_url = "https://opam.ocaml.org" + + lister = OpamLister(swh_scheduler, url=instance_url, instance="opam") + + mocked_popen = MagicMock() + mocked_popen.stdout = io.BytesIO(b"bar\nbaz\nfoo\n") + + # replaces the real Popen with a fake one + mocker.patch("swh.lister.opam.lister.Popen", return_value=mocked_popen) + + # call the lister and get all listed origins urls + stats = lister.run() + + assert stats.pages == 3 + assert stats.origins == 3 + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + expected_urls = [ + f"opam+{instance_url}/packages/bar/", + f"opam+{instance_url}/packages/baz/", + f"opam+{instance_url}/packages/foo/", + ] + + result_urls = [origin.url for origin in scheduler_origins] + + assert expected_urls == result_urls + + +def test_opam_binary(datadir, swh_scheduler): + + instance_url = f"file://{datadir}/fake_opam_repo" + + lister = OpamLister(swh_scheduler, url=instance_url, instance="fake") + + stats = lister.run() + + assert stats.pages == 4 + assert stats.origins == 4 + + scheduler_origins = swh_scheduler.get_listed_origins(lister.lister_obj.id).results + + expected_urls = [ + f"opam+{instance_url}/packages/agrid/", + f"opam+{instance_url}/packages/calculon/", + f"opam+{instance_url}/packages/directories/", + f"opam+{instance_url}/packages/ocb/", + ] + + result_urls = [origin.url for origin in scheduler_origins] + + assert expected_urls == result_urls diff --git a/swh/lister/opam/tests/test_tasks.py b/swh/lister/opam/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/opam/tests/test_tasks.py @@ -0,0 +1,31 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.lister.pattern import ListerStats + + +def test_opam_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.ping") + assert res + res.wait() + assert res.successful() + assert res.result == "OK" + + +def test_opam_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker): + # setup the mocked OpamLister + lister = mocker.patch("swh.lister.opam.tasks.OpamLister") + lister.from_configfile.return_value = lister + stats = ListerStats(pages=42, origins=42) + lister.run.return_value = stats + + res = swh_scheduler_celery_app.send_task("swh.lister.opam.tasks.OpamListerTask") + assert res + res.wait() + assert res.successful() + assert res.result == stats.dict() + + lister.from_configfile.assert_called_once_with() + lister.run.assert_called_once_with() diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -17,6 +17,7 @@ "gitea": {"url": "https://try.gitea.io/api/v1/",}, "tuleap": {"url": "https://tuleap.net",}, "gitlab": {"url": "https://gitlab.ow2.org/api/v4", "instance": "ow2",}, + "opam": {"url": "https://opam.ocaml.org", "instance": "opam"}, }