diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -60,6 +60,7 @@ lister.bitbucket=swh.lister.bitbucket:register lister.bower=swh.lister.bower:register lister.cgit=swh.lister.cgit:register + lister.conda=swh.lister.conda:register lister.cran=swh.lister.cran:register lister.crates=swh.lister.crates:register lister.debian=swh.lister.debian:register diff --git a/swh/lister/conda/__init__.py b/swh/lister/conda/__init__.py new file mode 100644 --- /dev/null +++ b/swh/lister/conda/__init__.py @@ -0,0 +1,19 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +""" +Conda lister +============ +""" + + +def register(): + from .lister import CondaLister + + return { + "lister": CondaLister, + "task_modules": ["%s.tasks" % __name__], + } diff --git a/swh/lister/conda/lister.py b/swh/lister/conda/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/conda/lister.py @@ -0,0 +1,150 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information +from collections import defaultdict +import datetime +import logging +from typing import Any, Dict, Iterator, List, Optional, Tuple + +import iso8601 +import requests +from tenacity.before_sleep import before_sleep_log + +from swh.lister.utils import throttling_retry +from swh.scheduler.interface import SchedulerInterface +from swh.scheduler.model import ListedOrigin + +from .. import __version__ +from ..pattern import CredentialsType, StatelessLister + +USER_AGENT = ( + f"Software Heritage Conda Lister v{__version__} " + "(+https://www.softwareheritage.org/contact)" +) + +logger = logging.getLogger(__name__) + +# Aliasing the page results returned by `get_pages` method from the lister. +CondaListerPage = Tuple[str, Dict[str, Dict[str, Any]]] + + +class CondaLister(StatelessLister[CondaListerPage]): + """List Conda (anaconda.com) origins.""" + + LISTER_NAME = "conda" + VISIT_TYPE = "conda" + INSTANCE = "conda" + BASE_REPO_URL = "https://repo.anaconda.com/pkgs" + REPO_URL_PATTERN = "{url}/{channel}/{arch}/repodata.json" + ORIGIN_URL_PATTERN = "https://anaconda.org/{channel}/{pkgname}" + ARCHIVE_URL_PATTERN = "{url}/{channel}/{arch}/{filename}" + + def __init__( + self, + scheduler: SchedulerInterface, + credentials: Optional[CredentialsType] = None, + url: str = BASE_REPO_URL, + channel: str = "", + archs: List = [], + ): + super().__init__( + scheduler=scheduler, + credentials=credentials, + instance=self.INSTANCE, + url=url, + ) + self.channel: str = channel + self.archs: List[str] = archs + self.packages: Dict[str, Any] = defaultdict(dict) + self.package_dates: Dict[str, Any] = defaultdict(list) + self.session = requests.Session() + self.session.headers.update( + { + "Accept": "application/json", + "User-Agent": USER_AGENT, + } + ) + + @throttling_retry(before_sleep=before_sleep_log(logger, logging.WARNING)) + def page_request(self, url: str, params: Dict[str, Any]) -> requests.Response: + + logger.debug("Fetching URL %s with params %s", url, params) + + response = self.session.get(url, params=params) + if response.status_code != 200: + logger.warning( + "Unexpected HTTP status code %s on %s: %s", + response.status_code, + response.url, + response.content, + ) + response.raise_for_status() + + return response + + def get_pages(self) -> Iterator[CondaListerPage]: + """Yield an iterator which returns 'page'""" + + for arch in self.archs: + repodata_url = self.REPO_URL_PATTERN.format( + url=self.url, channel=self.channel, arch=arch + ) + response = self.page_request(url=repodata_url, params={}) + packages = response.json()["packages"] + yield (arch, packages) + + def get_origins_from_page(self, page: CondaListerPage) -> Iterator[ListedOrigin]: + """Iterate on all pages and yield ListedOrigin instances.""" + assert self.lister_obj.id is not None + arch, packages = page + + for filename, entry in packages.items(): + artifact = { + "filename": filename, + "url": self.ARCHIVE_URL_PATTERN.format( + url=self.url, + channel=self.channel, + filename=filename, + arch=arch, + ), + "version": entry["version"], + } + if "md5" in entry: + checksums = {"md5": entry["md5"]} + elif "sha256" in entry: + checksums = {"sha256": entry["sha256"]} + else: + checksums = {} + + artifact["checksums"] = checksums + key = f"{arch}/{entry['version']}-{entry['build']}" + + self.packages[entry["name"]][key] = artifact + + if "timestamp" in entry: + dt = datetime.datetime.fromtimestamp( + entry["timestamp"] / 1e3, datetime.timezone.utc + ) + elif "date" in entry: + dt = iso8601.parse_date(entry["date"]) + else: + # TODO: Check if this case can exists + dt = None + last_update = None + if dt: + artifact["date"] = dt.isoformat() + self.package_dates[entry["name"]].append(dt) + last_update = max(self.package_dates[entry["name"]]) + + yield ListedOrigin( + lister_id=self.lister_obj.id, + visit_type=self.VISIT_TYPE, + url=self.ORIGIN_URL_PATTERN.format( + channel=self.channel, pkgname=entry["name"] + ), + last_update=last_update, + extra_loader_arguments={ + "artifacts": self.packages[entry["name"]], + }, + ) diff --git a/swh/lister/conda/tasks.py b/swh/lister/conda/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tasks.py @@ -0,0 +1,19 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.lister.conda.lister import CondaLister + + +@shared_task(name=__name__ + ".CondaListerTask") +def list_conda(**lister_args): + """Lister task for Anaconda registry""" + return CondaLister.from_configfile(**lister_args).run().dict() + + +@shared_task(name=__name__ + ".ping") +def _ping(): + return "OK" diff --git a/swh/lister/conda/tests/__init__.py b/swh/lister/conda/tests/__init__.py new file mode 100644 diff --git a/swh/lister/conda/tests/data/https_conda.anaconda.org/conda-forge_linux-64_repodata.json b/swh/lister/conda/tests/data/https_conda.anaconda.org/conda-forge_linux-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_conda.anaconda.org/conda-forge_linux-64_repodata.json @@ -0,0 +1,326 @@ +{ + "info": { + "subdir": "linux-64" + }, + "packages": { + "21cmfast-3.0.2-py36h1af98f8_1.tar.bz2": { + "build": "py36h1af98f8_1", + "build_number": 1, + "depends": [ + "_openmp_mutex >=4.5", + "astropy >=2.0", + "cached-property", + "cffi >=1.0", + "click", + "fftw >=3.3.8,<4.0a0", + "gsl >=2.6,<2.7.0a0", + "h5py >=2.8.0", + "libblas >=3.8.0,<4.0a0", + "libgcc-ng >=7.5.0", + "matplotlib-base", + "numpy", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "pyyaml", + "scipy" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "d65ab674acf3b7294ebacaec05fc5b54", + "name": "21cmfast", + "sha256": "1154fceeb5c4ee9bb97d245713ac21eb1910237c724d2b7103747215663273c2", + "size": 414494, + "subdir": "linux-64", + "timestamp": 1605110689658, + "version": "3.0.2" + }, + "gmsh-4.10.4-hc719622_0.tar.bz2": { + "build": "hc719622_0", + "build_number": 0, + "depends": [ + "fltk >=1.3.8,<1.4.0a0", + "gmp >=6.2.1,<7.0a0", + "jpeg >=9e,<10a", + "libblas >=3.9.0,<4.0a0", + "libgcc-ng >=12", + "libglu", + "liblapack >=3.9.0,<4.0a0", + "libpng >=1.6.37,<1.7.0a0", + "libstdcxx-ng >=12", + "libzlib >=1.2.12,<1.3.0a0", + "occt >=7.6.2,<7.7.0a0", + "xorg-libx11", + "xorg-libxext", + "xorg-libxfixes", + "xorg-libxmu", + "xorg-libxrender", + "zlib >=1.2.12,<1.3.0a0" + ], + "license": "GPL-2.0-or-later", + "license_family": "GPL", + "md5": "a49f34e7433422112dc5d62ea31a865d", + "name": "gmsh", + "sha256": "1520eb5ba06abe3e0e59f0dcc6d4b03b81efbf900b4f6aa54a37790d43eb5e3e", + "size": 165245778, + "subdir": "linux-64", + "timestamp": 1655651988256, + "version": "4.10.4" + }, + "lifetimes-0.11.1-py36h9f0ad1d_1.tar.bz2": { + "build": "py36h9f0ad1d_1", + "build_number": 1, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "faa398f7ba0d60ce44aa6eeded490cee", + "name": "lifetimes", + "sha256": "f82a352dfae8abceeeaa538b220fd9c5e4aa4e59092a6a6cea70b9ec0581ea03", + "size": 492612, + "subdir": "linux-64", + "timestamp": 1594037976425, + "version": "0.11.1" + }, + "lifetimes-0.11.1-py36hc560c46_1.tar.bz2": { + "build": "py36hc560c46_1", + "build_number": 1, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "pypy3.6 >=7.3.1", + "python >=3.6,<3.7.0a0", + "python_abi 3.6 *_pypy36_pp73", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "c53a689a4c5948e84211bdfc23e3fe68", + "name": "lifetimes", + "sha256": "76146c2ebd6e3b65928bde53a2585287759d77beba785c0eeb889ee565c0035d", + "size": 503895, + "subdir": "linux-64", + "timestamp": 1594037977032, + "version": "0.11.1" + }, + "lifetimes-0.11.1-py37_0.tar.bz2": { + "build": "py37_0", + "build_number": 0, + "constrains": [ + "python_abi * *_cp37m" + ], + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.7,<3.8.0a0", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "8747583c5ab10e29e11a36715160ac1f", + "name": "lifetimes", + "sha256": "b527ec13e6b7e1f051c1cbb59b4e72e8bc9a6e1cdc15d37e3202303ec4783343", + "size": 502789, + "subdir": "linux-64", + "timestamp": 1582057777048, + "version": "0.11.1" + }, + "lifetimes-0.11.1-py37hc8dfbb8_1.tar.bz2": { + "build": "py37hc8dfbb8_1", + "build_number": 1, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.7,<3.8.0a0", + "python_abi 3.7.* *_cp37m", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "db90bd9e692ca33e3ff8f5cdf58e1e47", + "name": "lifetimes", + "sha256": "deb3ec23a982f88277158e4d2c02bc48202f149b6df795f5cc79682a6d99d649", + "size": 502337, + "subdir": "linux-64", + "timestamp": 1594037970894, + "version": "0.11.1" + }, + "lifetimes-0.11.1-py38_0.tar.bz2": { + "build": "py38_0", + "build_number": 0, + "constrains": [ + "python_abi * *_cp38" + ], + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.8,<3.9.0a0", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "92542835a454c9387e735052a6042b2f", + "name": "lifetimes", + "sha256": "964638f08e838daa9aa524ddcc9d6edcb47d6432a69e214076d33d16e3cebad7", + "size": 494618, + "subdir": "linux-64", + "timestamp": 1582063471070, + "version": "0.11.1" + }, + "lifetimes-0.11.1-py38h32f6830_1.tar.bz2": { + "build": "py38h32f6830_1", + "build_number": 1, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.8,<3.9.0a0", + "python_abi 3.8.* *_cp38", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "96a57bade6d4117cf0107bc5fbca72e3", + "name": "lifetimes", + "sha256": "99903068de43970c1fa6913baf7b061b7eecc3c79f861819174c50656953aaaf", + "size": 504750, + "subdir": "linux-64", + "timestamp": 1594037981828, + "version": "0.11.1" + }, + "lifetimes-0.11.3-py36h5fab9bb_1.tar.bz2": { + "build": "py36h5fab9bb_1", + "build_number": 1, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "1545187174973addf474b77af257795c", + "name": "lifetimes", + "sha256": "77218996fde7cf3a22bbb19359cbe67d2f1ccdbae662a35fbefe819d6869d296", + "size": 496046, + "subdir": "linux-64", + "timestamp": 1604358492154, + "version": "0.11.3" + }, + "lifetimes-0.11.3-py36h9f0ad1d_0.tar.bz2": { + "build": "py36h9f0ad1d_0", + "build_number": 0, + "depends": [ + "autograd >=1.2.0", + "dill >=0.2.6", + "numpy >=1.10.0", + "pandas >=0.24.0", + "python >=3.6,<3.7.0a0", + "python_abi 3.6.* *_cp36m", + "scipy >=1.0.0", + "setuptools" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "0ac8686196b61e31dbf110faf98cdfba", + "name": "lifetimes", + "sha256": "c7c9c766f749f1931588d7c9eccba35e25bcb3aef16be897ece459108213aad9", + "size": 495338, + "subdir": "linux-64", + "timestamp": 1594081612442, + "version": "0.11.3" + }, + "r-getpass-0.2_2-r35h516909a_1003.tar.bz2": { + "build": "r35h516909a_1003", + "build_number": 1003, + "depends": [ + "libgcc-ng >=7.3.0", + "r-base >=3.5,<3.6.0a0", + "r-rstudioapi >=0.5" + ], + "license": "BSD 2-clause", + "license_family": "BSD", + "md5": "5e29a4deb9e9c1dd7b373169bd21b7ba", + "name": "r-getpass", + "sha256": "d92f307784c0a9c1300b6fcd41f83306d9c329a6c74f8f99b9daa1f5e660f97c", + "size": 240032, + "subdir": "linux-64", + "timestamp": 1563494129797, + "version": "0.2_2" + }, + "xeus-python-0.5.1-py27hc9558a2_0.tar.bz2": { + "build": "py27hc9558a2_0", + "build_number": 0, + "constrains": [ + "python_abi * *_cp27mu" + ], + "depends": [ + "jedi >=0.13.1,<0.14", + "libgcc-ng >=7.3.0", + "libstdcxx-ng >=7.3.0", + "pygments >=2.3.1,<3.0.0", + "python >=2.7,<2.8.0a0", + "xeus >=0.23.2,<0.24.0a0", + "zeromq >=4.3.2,<4.4.0a0" + ], + "license": "BSD-3-Clause", + "license_family": "BSD", + "md5": "35d745507ed6b4b70be802a01d1e366c", + "name": "xeus-python", + "sha256": "37b29674ce9d626a0056aedf6690ea76f8ce29e6bf381081a6571400ea756a2f", + "size": 219071, + "subdir": "linux-64", + "timestamp": 1568284339855, + "version": "0.5.1" + }, + "zziplib-0.13.69-hed695b0_1.tar.bz2": { + "build": "hed695b0_1", + "build_number": 1, + "depends": [ + "libgcc-ng >=7.3.0", + "zlib >=1.2.11,<1.3.0a0" + ], + "license": "GPL-2.0", + "license_family": "GPL", + "md5": "a41a0433db5b33992226d2f9c3076e87", + "name": "zziplib", + "sha256": "217ce1b813b1273fa2ff379ef1c1fb8a1d281133174d6f834efff054bc714746", + "size": 101482, + "subdir": "linux-64", + "timestamp": 1586464758560, + "version": "0.13.69" + } + }, + "packages.conda": {}, + "removed": [ + "abinit-8.10.2-h279e56a_1.tar.bz2", + "xwidgets-0.20.2-hc9558a2_0.tar.bz2", + "yaml-0.2.5-h7f98852_1.tar.bz2" + ], + "repodata_version": 1 +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_linux-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_linux-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_linux-64_repodata.json @@ -0,0 +1,78 @@ +{ + "info": { + "arch": "x86_64", + "platform": "linux", + "subdir": "linux-64" + }, + "packages": { + "xlrd-0.9.4-py27_0.tar.bz2": { + "build": "py27_0", + "build_number": 0, + "date": "2015-07-20", + "depends": [ + "python 2.7*" + ], + "license": "BSD", + "md5": "73f17f03d3d10458b2b697274e0f0216", + "name": "xlrd", + "size": 182911, + "version": "0.9.4" + }, + "xlsxwriter-0.5.2-py27_0.tar.bz2": { + "build": "py27_0", + "build_number": 0, + "date": "2014-02-17", + "depends": [ + "python 2.7*" + ], + "license": "BSD", + "md5": "b2dbc8357f0e50eb6d3478fb0fc576b7", + "name": "xlsxwriter", + "size": 164618, + "version": "0.5.2" + }, + "xlsxwriter-0.5.2-py33_0.tar.bz2": { + "build": "py33_0", + "build_number": 0, + "date": "2014-02-17", + "depends": [ + "python 3.3*" + ], + "license": "BSD", + "md5": "d51ef8f39c1bde10899d9a00c1978694", + "name": "xlsxwriter", + "size": 172040, + "version": "0.5.2" + }, + "xlsxwriter-0.6.5-py26_0.tar.bz2": { + "build": "py26_0", + "build_number": 0, + "date": "2015-01-05", + "depends": [ + "python 2.6*" + ], + "license": "BSD", + "md5": "00dcde80c9e9401f522819ce54cf0749", + "name": "xlsxwriter", + "size": 178349, + "version": "0.6.5" + }, + "zope.sqlalchemy-0.7.7-py36_0.tar.bz2": { + "build": "py36_0", + "build_number": 0, + "date": "2016-12-22", + "depends": [ + "python 3.6*", + "sqlalchemy >=0.5.1", + "transaction", + "zope.interface >=3.6.0" + ], + "license": "ZPL 2.1", + "license_family": "Other", + "md5": "753f399dbb0c04d484b751a96d623e58", + "name": "zope.sqlalchemy", + "size": 28261, + "version": "0.7.7" + } + } +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_osx-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_osx-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_osx-64_repodata.json @@ -0,0 +1,56 @@ +{ + "info": { + "arch": "x86_64", + "platform": "osx", + "subdir": "osx-64" + }, + "packages": { + "zope.interface-4.4.1-py35_0.tar.bz2": { + "build": "py35_0", + "build_number": 0, + "date": "2017-05-15", + "depends": [ + "python 3.5*", + "zope" + ], + "license": "ZPL 2.1", + "license_family": "Other", + "md5": "896deacca630c01a80269a9022ee3154", + "name": "zope.interface", + "size": 205493, + "version": "4.4.1" + }, + "zope.interface-4.4.1-py36_0.tar.bz2": { + "build": "py36_0", + "build_number": 0, + "date": "2017-05-15", + "depends": [ + "python 3.6*", + "zope" + ], + "license": "ZPL 2.1", + "license_family": "Other", + "md5": "e62f3f809e4ecc4290c9cd75f752a119", + "name": "zope.interface", + "size": 204965, + "version": "4.4.1" + }, + "zope.sqlalchemy-0.7.7-py36_0.tar.bz2": { + "build": "py36_0", + "build_number": 0, + "date": "2016-12-24", + "depends": [ + "python 3.6*", + "sqlalchemy >=0.5.1", + "transaction", + "zope.interface >=3.6.0" + ], + "license": "ZPL 2.1", + "license_family": "Other", + "md5": "1273c9f8e1c70fd49806f1ffaec26ad0", + "name": "zope.sqlalchemy", + "size": 27722, + "version": "0.7.7" + } + } +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_win-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_win-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_free_win-64_repodata.json @@ -0,0 +1,298 @@ +{ + "info": { + "arch": "x86_64", + "platform": "win", + "subdir": "win-64" + }, + "packages": { + "mathjax-2.2-0.tar.bz2": { + "build": "0", + "build_number": 0, + "date": "2014-03-28", + "depends": [], + "license": "Apache", + "md5": "6815d38d290892f71008721e1965eb5f", + "name": "mathjax", + "size": 7392529, + "version": "2.2" + }, + "matplotlib-1.1.1-np16py27_2.tar.bz2": { + "build": "np16py27_2", + "build_number": 2, + "date": "2012-11-08", + "depends": [ + "dateutil", + "numpy 1.6*", + "pyside 1.1.2", + "python 2.7*", + "pytz" + ], + "license": "PSF-based", + "license_family": "PSF", + "md5": "3ff9563391d732c99b3d6c3f2f1dca01", + "name": "matplotlib", + "size": 27445455, + "version": "1.1.1" + }, + "matplotlib-1.4.3-np19py26_0.tar.bz2": { + "build": "np19py26_0", + "build_number": 0, + "date": "2015-02-19", + "depends": [ + "dateutil", + "numpy 1.9*", + "pyparsing 2.0.3", + "pyqt 4.*", + "python 2.6*", + "pytz" + ], + "license": "PSF-based", + "license_family": "PSF", + "md5": "f9ca1cc79f387ca66e788b7f7cd7d805", + "name": "matplotlib", + "size": 44754024, + "version": "1.4.3" + }, + "matplotlib-1.4.3-np19py26_1.tar.bz2": { + "build": "np19py26_1", + "build_number": 1, + "date": "2015-03-19", + "depends": [ + "numpy 1.9*", + "pyparsing 2.0.3", + "pyqt 4.*", + "python 2.6*", + "python-dateutil", + "pytz" + ], + "license": "PSF-based", + "license_family": "PSF", + "md5": "c431dfd168b37f54df73e86047af2904", + "name": "matplotlib", + "size": 44753969, + "version": "1.4.3" + }, + "matplotlib-1.4.3-np19py27_0.tar.bz2": { + "build": "np19py27_0", + "build_number": 0, + "date": "2015-02-19", + "depends": [ + "dateutil", + "numpy 1.9*", + "pyparsing 2.0.3", + "pyqt 4.*", + "python 2.7*", + "pytz" + ], + "license": "PSF-based", + "license_family": "PSF", + "md5": "b853e7c7335148d275fd2b405acd3f2e", + "name": "matplotlib", + "size": 44772871, + "version": "1.4.3" + }, + "matplotlib-1.4.3-np19py27_1.tar.bz2": { + "build": "np19py27_1", + "build_number": 1, + "date": "2015-03-19", + "depends": [ + "numpy 1.9*", + "pyparsing 2.0.3", + "pyqt 4.*", + "python 2.7*", + "python-dateutil", + "pytz" + ], + "license": "PSF-based", + "license_family": "PSF", + "md5": "1d9aa1487ccc352d525fe2245629a603", + "name": "matplotlib", + "size": 44770821, + "version": "1.4.3" + }, + "sympy-1.0-py35_0.tar.bz2": { + "build": "py35_0", + "build_number": 0, + "date": "2016-03-09", + "depends": [ + "mpmath >=0.19", + "python 3.5*" + ], + "license": "3-clause BSD", + "license_family": "BSD", + "md5": "9594945afa3fa1a82300810042f0f5ca", + "name": "sympy", + "size": 6719248, + "version": "1.0" + }, + "theano-0.8.2-py27_0.tar.bz2": { + "build": "py27_0", + "build_number": 0, + "date": "2016-10-15", + "depends": [ + "libpython >=2.0", + "m2w64-toolchain", + "mkl-service", + "numpy >=1.3.0", + "python 2.7*", + "scipy >=0.7.0", + "six >=1.9.0" + ], + "license": "BSD 3-Clause", + "license_family": "BSD", + "md5": "847f294850c6c38fc6c5d80802348163", + "name": "theano", + "size": 3774182, + "version": "0.8.2" + }, + "theano-0.8.2-py34_0.tar.bz2": { + "build": "py34_0", + "build_number": 0, + "date": "2016-10-15", + "depends": [ + "libpython >=2.0", + "m2w64-toolchain", + "mkl-service", + "numpy >=1.3.0", + "python 3.4*", + "scipy >=0.7.0", + "six >=1.9.0" + ], + "license": "BSD 3-Clause", + "license_family": "BSD", + "md5": "7d398562657e0698933d7f2563af3495", + "name": "theano", + "size": 3856268, + "version": "0.8.2" + }, + "tornado-3.2.2-py34_0.tar.bz2": { + "build": "py34_0", + "build_number": 0, + "date": "2014-06-27", + "depends": [ + "python 3.4*", + "ssl_match_hostname 3.4.0.2" + ], + "license": "Apache 2.0", + "license_family": "Apache", + "md5": "da3c3d9200368c67a255c388960b4acd", + "name": "tornado", + "size": 597297, + "version": "3.2.2" + }, + "tornado-4.0.1-py26_0.tar.bz2": { + "build": "py26_0", + "build_number": 0, + "date": "2014-08-15", + "depends": [ + "python 2.6*", + "ssl_match_hostname 3.4.0.2" + ], + "license": "Apache 2.0", + "license_family": "Apache", + "md5": "1a846e735220547df80e52df0bfdbeb9", + "name": "tornado", + "size": 460262, + "version": "4.0.1" + }, + "toro-1.0.1-py35_0.tar.bz2": { + "build": "py35_0", + "build_number": 0, + "date": "2016-08-29", + "depends": [ + "python 3.5*", + "tornado >=3" + ], + "license": "Apache Software License", + "license_family": "Apache", + "md5": "91a37313f5a837a0764a0c6d1ab8f6f0", + "name": "toro", + "size": 17017, + "version": "1.0.1" + }, + "toro-1.0.1-py36_0.tar.bz2": { + "build": "py36_0", + "build_number": 0, + "date": "2017-07-20", + "depends": [ + "python 3.6*", + "tornado >=3" + ], + "license": "Apache Software License", + "license_family": "Apache", + "md5": "0046c19a85a7e837c6c8ff88f0c59574", + "name": "toro", + "size": 20044, + "version": "1.0.1" + }, + "xlsxwriter-0.9.2-py27_0.tar.bz2": { + "build": "py27_0", + "build_number": 0, + "date": "2016-06-20", + "depends": [ + "python 2.7*" + ], + "license": "BSD", + "md5": "ad3fd677c06cf258b2fa809c1594c95d", + "name": "xlsxwriter", + "size": 195974, + "version": "0.9.2" + }, + "xlsxwriter-0.9.2-py34_0.tar.bz2": { + "build": "py34_0", + "build_number": 0, + "date": "2016-06-20", + "depends": [ + "python 3.4*" + ], + "license": "BSD", + "md5": "3f1aae9666e85040b91fe8816e2b3e6a", + "name": "xlsxwriter", + "size": 197701, + "version": "0.9.2" + }, + "xlsxwriter-0.9.2-py35_0.tar.bz2": { + "build": "py35_0", + "build_number": 0, + "date": "2016-06-20", + "depends": [ + "python 3.5*" + ], + "license": "BSD", + "md5": "c3c9e555e776bc2188b5352d94d3f209", + "name": "xlsxwriter", + "size": 196148, + "version": "0.9.2" + }, + "xlwt-0.7.4-py26_0.tar.bz2": { + "build": "py26_0", + "build_number": 0, + "date": "2013-02-20", + "depends": [ + "python 2.6*" + ], + "license": "BSD", + "md5": "86d6d7699c9b91e37124504452c5a89f", + "name": "xlwt", + "size": 174343, + "version": "0.7.4" + }, + "zope.sqlalchemy-0.7.7-py36_0.tar.bz2": { + "build": "py36_0", + "build_number": 0, + "date": "2016-12-27", + "depends": [ + "python 3.6*", + "sqlalchemy >=0.5.1", + "transaction", + "zope.interface >=3.6.0" + ], + "license": "ZPL 2.1", + "license_family": "Other", + "md5": "93239b00983bf329ca50d86280a17280", + "name": "zope.sqlalchemy", + "size": 28421, + "version": "0.7.7" + } + } +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_main_linux-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_main_linux-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_main_linux-64_repodata.json @@ -0,0 +1,163 @@ +{ + "info": { + "subdir": "linux-64" + }, + "packages": { + "traits-5.1.2-py36h7b6447c_0.conda": { + "build": "py36h7b6447c_0", + "build_number": 0, + "depends": [ + "libgcc-ng >=7.3.0", + "python >=3.6,<3.7.0a0" + ], + "license": "BSD 3-clause", + "md5": "d5e5d82b3b39c99482bea519025bf874", + "name": "traits", + "sha256": "3021f5a1623c21191f2bdbed26a88ef22583c5895f02b271bbbb7f2b21a147a3", + "size": 406636, + "subdir": "linux-64", + "timestamp": 1569359641034, + "version": "5.1.2" + }, + "trio-0.19.0-py38h06a4308_0.conda": { + "build": "py38h06a4308_0", + "build_number": 0, + "depends": [ + "async_generator >=1.9", + "attrs >=19.2.0", + "idna", + "outcome", + "python >=3.8,<3.9.0a0", + "sniffio", + "sortedcontainers" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "3d41f81c348fd43f7c208429bc24bfd9", + "name": "trio", + "sha256": "99bc0c77b47c939546af4fcac0fd05dd0c3f0bf90533b25afe786739bb93cde2", + "size": 472387, + "subdir": "linux-64", + "timestamp": 1644385298098, + "version": "0.19.0" + }, + "trio-0.19.0-py39h06a4308_0.conda": { + "build": "py39h06a4308_0", + "build_number": 0, + "depends": [ + "async_generator >=1.9", + "attrs >=19.2.0", + "idna", + "outcome", + "python >=3.9,<3.10.0a0", + "sniffio", + "sortedcontainers" + ], + "license": "MIT", + "license_family": "MIT", + "md5": "973450d5dc748e9b3c73d5de739bda4e", + "name": "trio", + "sha256": "5ec5995fa28d4be063ea098813eba24b4aed43c7d095c96309511f29364a843f", + "size": 476600, + "subdir": "linux-64", + "timestamp": 1644387179434, + "version": "0.19.0" + }, + "trollius-2.2-py27h14c3975_0.conda": { + "build": "py27h14c3975_0", + "build_number": 0, + "depends": [ + "futures", + "libgcc-ng >=7.2.0", + "python >=2.7,<2.8.0a0", + "six" + ], + "license": "Apache-2.0", + "license_family": "Apache", + "md5": "cce3fa393968970df90e6bbb7fc543d4", + "name": "trollius", + "sha256": "1e66c1f43eae93493522aac0e0afe2350908405815a1a681d2e6ee36f584916b", + "size": 183031, + "subdir": "linux-64", + "timestamp": 1522776279053, + "version": "2.2" + }, + "trollius-2.2-py35h14c3975_0.conda": { + "build": "py35h14c3975_0", + "build_number": 0, + "depends": [ + "libgcc-ng >=7.2.0", + "python >=3.5,<3.6.0a0", + "six" + ], + "license": "Apache-2.0", + "license_family": "Apache", + "md5": "06acd4ffb1aed4e338f9d2fd846bbdf7", + "name": "trollius", + "sha256": "e59e77ccf6a8c00f4556879610bb6bc70e44ccf23981c60394d88c97d21fd496", + "size": 188796, + "subdir": "linux-64", + "timestamp": 1522776396637, + "version": "2.2" + }, + "trollius-2.2-py36h14c3975_0.conda": { + "build": "py36h14c3975_0", + "build_number": 0, + "depends": [ + "libgcc-ng >=7.2.0", + "python >=3.6,<3.7.0a0", + "six" + ], + "license": "Apache-2.0", + "license_family": "Apache", + "md5": "1869094c6120aeb8a750d9977b58ec76", + "name": "trollius", + "sha256": "771b15c00e5a53a0ed33933913bb622093efed2024915613b0e2563b855d6a59", + "size": 184557, + "subdir": "linux-64", + "timestamp": 1522776395895, + "version": "2.2" + }, + "zope.interface-5.2.0-py37h27cfd23_0.conda": { + "build": "py37h27cfd23_0", + "build_number": 0, + "depends": [ + "libgcc-ng >=7.3.0", + "python >=3.7,<3.8.0a0", + "setuptools", + "zope" + ], + "license": "ZPL-2.1", + "license_family": "Other", + "md5": "60b9acc040fb34f1b58edc7007098cb4", + "name": "zope.interface", + "sha256": "08fe40622ddc800597db64c3026c1fc059594b8179cdb750a973663c7987fd0b", + "size": 284459, + "subdir": "linux-64", + "timestamp": 1606940275034, + "version": "5.2.0" + }, + "zstd-static-1.4.5-he6710b0_0.conda": { + "build": "he6710b0_0", + "build_number": 0, + "depends": [ + "libgcc-ng >=7.3.0", + "libstdcxx-ng >=7.3.0", + "zstd 1.4.5 h9ceee32_0" + ], + "license": "BSD 3-Clause", + "md5": "e82bd6f2f8a8ca682832681581d2611d", + "name": "zstd-static", + "sha256": "c3f25e5cdb977aca6ad96fe70581c04d7f037784780d9a98812949e092dd84da", + "size": 306282, + "subdir": "linux-64", + "timestamp": 1595964895392, + "version": "1.4.5" + } + }, + "removed": [ + "numpy-devel-1.14.3-py36ha22f7c6_2.conda", + "numpy-devel-1.14.3-py36ha22f7c6_2.tar.bz2" + ], + "repodata_version": 1 +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-32_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-32_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-32_repodata.json @@ -0,0 +1,187 @@ +{ + "info": { + "subdir": "win-32" + }, + "packages": { + "m2w64-libssh2-1.7.0-1.tar.bz2": { + "build": "1", + "build_number": 1, + "depends": [ + "m2w64-openssl", + "m2w64-zlib", + "msys2-conda-epoch >=20160418" + ], + "license": "BSD", + "md5": "b3da43724f521d252cdc108faa7bc807", + "name": "m2w64-libssh2", + "sha256": "a93eaca5fd385a468acc25ad33b5d54c9607fe4f388e093f19657fb8485b55b7", + "size": 280923, + "subdir": "win-32", + "version": "1.7.0" + }, + "m2w64-libssh2-1.7.0-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2w64-openssl", + "m2w64-zlib", + "msys2-conda-epoch >=20160418" + ], + "license": "BSD", + "md5": "8d0be733f4e5b380f0daf34b8d2cab61", + "name": "m2w64-libssh2", + "sha256": "65b85bde00b7caf6fa711eb77e9f048664286c9b98f02f2366cc0096a2c8f2b4", + "size": 266020, + "subdir": "win-32", + "version": "1.7.0" + }, + "m2w64-libsystre-1.0.1-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2w64-libtre-git", + "msys2-conda-epoch >=20160418" + ], + "license": "BSD", + "md5": "f86b6490fafe35ec3a6bc85e733354d0", + "name": "m2w64-libsystre", + "sha256": "5869f68e70a2829107b6f433df723f04029f835d15a88a459fee06b96f3cd0c5", + "size": 12450, + "subdir": "win-32", + "version": "1.0.1" + }, + "m2w64-libsystre-1.0.1-3.tar.bz2": { + "build": "3", + "build_number": 3, + "depends": [ + "m2w64-libtre-git", + "msys2-conda-epoch >=20160418" + ], + "license": "BSD", + "md5": "d1876bbc2c28c96fdbd9968d685632df", + "name": "m2w64-libsystre", + "sha256": "30d9b90d2613d57e2f8cafa8cbb244e38d1dc8df28f86c2e37d8c52867d2fd1a", + "size": 13080, + "subdir": "win-32", + "version": "1.0.1" + }, + "m2w64-libwinpthread-git-5.0.0.4634.697f757-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "msys2-conda-epoch >=20160418" + ], + "license": "MIT, BSD", + "md5": "4b5eceb31f7588d60ae4a2b9e157b094", + "name": "m2w64-libwinpthread-git", + "sha256": "7dc75a399d3e40c24addc7594ffe5c5697182a84a9c255659198c674a5f94eaa", + "size": 33384, + "subdir": "win-32", + "version": "5.0.0.4634.697f757" + }, + "m2w64-libxml2-2.9.3-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2w64-gcc-libs", + "m2w64-gettext", + "m2w64-xz", + "m2w64-zlib", + "msys2-conda-epoch >=20160418" + ], + "license": "MIT", + "md5": "7713bd630cd0d265ca08b61edb581cd6", + "name": "m2w64-libxml2", + "sha256": "6ad45640a5e7b669a42788e2b87c64d1a3b53c92410a79a1498ce6db57e43f75", + "size": 2201556, + "subdir": "win-32", + "version": "2.9.3" + }, + "m2w64-libxml2-2.9.3-3.tar.bz2": { + "build": "3", + "build_number": 3, + "depends": [ + "m2w64-gcc-libs", + "m2w64-gettext", + "m2w64-xz", + "m2w64-zlib", + "msys2-conda-epoch >=20160418" + ], + "license": "MIT", + "md5": "aef8c2da1a34a37216e9ad11b9ed4811", + "name": "m2w64-libxml2", + "sha256": "30b18de8e9f62a332dd42aa6802fbbbc40407cd0e33968d04dee7dc6170a067a", + "size": 2180717, + "subdir": "win-32", + "version": "2.9.3" + }, + "m2w64-libxml2-2.9.3-4.tar.bz2": { + "build": "4", + "build_number": 4, + "depends": [ + "m2w64-gcc-libs", + "m2w64-gettext", + "m2w64-xz", + "m2w64-zlib", + "msys2-conda-epoch >=20160418" + ], + "license": "MIT", + "md5": "e98699a40bca94a2bfa050050f71d63b", + "name": "m2w64-libxml2", + "sha256": "b3b0982fb946f4e69b20fbece73c924d6e157590fd7d174e95def758a1051ee0", + "size": 2206122, + "subdir": "win-32", + "version": "2.9.3" + }, + "m2w64-make-4.1.2351.a80a8b8-1.tar.bz2": { + "build": "1", + "build_number": 1, + "depends": [ + "msys2-conda-epoch >=20160418" + ], + "license": "GPL3", + "md5": "a91303355b053ba3eea7795b0ad79771", + "name": "m2w64-make", + "sha256": "c25d106bec45e92d3b0fe35ed96d8948d81f66f4f8457d8f8c34140499ec0640", + "size": 119132, + "subdir": "win-32", + "version": "4.1.2351.a80a8b8" + }, + "m2w64-make-4.1.2351.a80a8b8-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "msys2-conda-epoch >=20160418" + ], + "license": "GPL3", + "md5": "c49e37168141611750a1a0a9f9f9a601", + "name": "m2w64-make", + "sha256": "46006ff6f403e190618c058965f9d87f86e0a64ac1e4fdad4f1d5cc55b99714c", + "size": 120036, + "subdir": "win-32", + "version": "4.1.2351.a80a8b8" + }, + "posix-1.0.0-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2-base", + "m2-diffutils", + "m2-make", + "m2-tar", + "m2-unzip", + "m2-zip", + "msys2-conda-epoch >=20160418" + ], + "md5": "4c042b2e69064d52f6865e3a6ab4f840", + "name": "posix", + "sha256": "5e931c62b89821272b3f70fb2323805f4df9802f2c2cc0c07776fa3f77d83351", + "size": 2144, + "subdir": "win-32", + "version": "1.0.0" + } + }, + "packages.conda": {}, + "removed": [], + "repodata_version": 1 +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_msys2_win-64_repodata.json @@ -0,0 +1,124 @@ +{ + "info": { + "subdir": "win-64" + }, + "packages": { + "m2-autoconf-2.69-3.tar.bz2": { + "build": "3", + "build_number": 3, + "depends": [ + "m2-bash", + "m2-diffutils", + "m2-gawk", + "m2-m4", + "m2-perl", + "msys2-conda-epoch >=20160418" + ], + "license": "GPL, GPL2, GPL3, custom", + "md5": "480ef475fcb746cd881ea117b50e4ddb", + "name": "m2-autoconf", + "namespace": "m2", + "sha256": "debfd382a6727f3b1622004acbb294ace1805a142fb86b11edf31cf68579aeec", + "size": 743432, + "subdir": "win-64", + "version": "2.69" + }, + "m2-autoconf-2.69-4.tar.bz2": { + "build": "4", + "build_number": 4, + "depends": [ + "m2-bash", + "m2-diffutils", + "m2-gawk", + "m2-m4", + "m2-perl", + "msys2-conda-epoch >=20160418" + ], + "license": "GPL, GPL2, GPL3, custom", + "md5": "45caf1c00748798ca7d801d6611cafb2", + "name": "m2-autoconf", + "namespace": "m2", + "sha256": "a60a83651b5123881fb129e88a9c74e390d054fab08f1ce77528a5edc6d89c42", + "size": 743827, + "subdir": "win-64", + "version": "2.69" + }, + "m2-libguile-2.0.11-3.tar.bz2": { + "build": "3", + "build_number": 3, + "depends": [ + "m2-gmp", + "m2-libffi", + "m2-libgc", + "m2-libltdl", + "m2-libunistring", + "m2-ncurses", + "msys2-conda-epoch >=20160418" + ], + "license": "GPL", + "md5": "1463c2a71ac0b5d8aef0c09058aeaf9e", + "name": "m2-libguile", + "sha256": "014e3a983be9c4753ea1a321cc4ac09c8728f171a230a143afcf43ed80700052", + "size": 2854189, + "subdir": "win-64", + "version": "2.0.11" + }, + "m2-libguile-2.0.11-4.tar.bz2": { + "build": "4", + "build_number": 4, + "depends": [ + "m2-gmp", + "m2-libffi", + "m2-libgc", + "m2-libltdl", + "m2-libunistring", + "m2-ncurses", + "msys2-conda-epoch >=20160418" + ], + "license": "GPL", + "md5": "2e9ab3ea9db82122b16037ed3e5a8c6a", + "name": "m2-libguile", + "sha256": "79bc27af0ffa0f378b102fd1823fc5080f2573c4fac5c8094631d07b56f43a92", + "size": 2832219, + "subdir": "win-64", + "version": "2.0.11" + }, + "m2-libiconv-1.14-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2-gcc-libs", + "msys2-conda-epoch >=20160418" + ], + "license": "GPL, LGPL", + "md5": "7cc3fa8e43a3a793311fe937746d0fb1", + "name": "m2-libiconv", + "sha256": "a44fcbf7b25bd4eb0fc26e27760d6db520af807518d67f21f9b812aff6f09ed4", + "size": 732060, + "subdir": "win-64", + "version": "1.14" + }, + "posix-1.0.0-2.tar.bz2": { + "build": "2", + "build_number": 2, + "depends": [ + "m2-base", + "m2-diffutils", + "m2-make", + "m2-tar", + "m2-unzip", + "m2-zip", + "msys2-conda-epoch >=20160418" + ], + "md5": "83ce222b94ccaf79838a6ffa5a1cd6bc", + "name": "posix", + "sha256": "da38a94980b046a6de7eff338c6a880111650f35cd9b535735ec64ddd750221f", + "size": 2162, + "subdir": "win-64", + "version": "1.0.0" + } + }, + "packages.conda": {}, + "removed": [], + "repodata_version": 1 +} diff --git a/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_pro_linux-64_repodata.json b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_pro_linux-64_repodata.json new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/data/https_repo.anaconda.com/pkgs_pro_linux-64_repodata.json @@ -0,0 +1,108 @@ +{ + "info": { + "arch": "x86_64", + "platform": "linux", + "subdir": "linux-64" + }, + "packages": { + "numexpr-2.3.1-np18py34_p0.tar.bz2": { + "build": "np18py34_p0", + "build_number": 0, + "date": "2014-05-06", + "depends": [ + "mkl-rt 11.1", + "numpy 1.8*", + "python 3.4*" + ], + "features": "mkl", + "license": "MIT", + "md5": "371e9b879ba32ff6bb785440c6417dcc", + "name": "numexpr", + "pub_date": "2014-05-06", + "size": 322734, + "version": "2.3.1" + }, + "numexpr-2.3.1-np19py26_p0.tar.bz2": { + "build": "np19py26_p0", + "build_number": 0, + "date": "2014-09-10", + "depends": [ + "mkl-rt 11.1", + "numpy 1.9*", + "python 2.6*" + ], + "features": "mkl", + "license": "MIT", + "md5": "f0104039a0092d5f2260aa6d39c2179a", + "name": "numexpr", + "pub_date": "2014-09-10", + "size": 324458, + "version": "2.3.1" + }, + "numpy-1.6.2-py27_pro0.tar.bz2": { + "build": "py27_pro0", + "build_number": 0, + "date": "2012-11-09", + "depends": [ + "blas * mkl", + "mkl 10.3", + "nose", + "python 2.7*" + ], + "features": "mkl", + "license": "BSD", + "md5": "5c6a24c424112f0d64e87d057fce4b25", + "name": "numpy", + "size": 4362029, + "version": "1.6.2" + }, + "numpy-1.7.0-py26_p0.tar.bz2": { + "build": "py26_p0", + "build_number": 0, + "date": "2013-03-04", + "depends": [ + "blas * mkl", + "mkl 10.3", + "nose", + "python 2.6*" + ], + "features": "mkl", + "license": "BSD", + "md5": "b7e31d6af95e98aee4bf1fbdaafed91c", + "name": "numpy", + "pub_date": "2013-03-04", + "size": 4820431, + "version": "1.7.0" + }, + "wiserf-1.1-np17py27_0.tar.bz2": { + "build": "np17py27_0", + "build_number": 0, + "date": "2012-11-19", + "depends": [ + "numpy 1.7*", + "python 2.7*", + "scipy" + ], + "license_family": "Proprietary", + "md5": "1fa4a73df5d52567eabee0833d57a6ba", + "name": "wiserf", + "size": 1135464, + "version": "1.1" + }, + "wiserf-1.1-np17py27_1.tar.bz2": { + "build": "np17py27_1", + "build_number": 1, + "date": "2013-01-22", + "depends": [ + "numpy 1.7*", + "python 2.7*", + "scipy" + ], + "license_family": "Proprietary", + "md5": "96d0b5c834a5a9c0251b9956d87cd182", + "name": "wiserf", + "size": 1920347, + "version": "1.1" + } + } +} diff --git a/swh/lister/conda/tests/test_lister.py b/swh/lister/conda/tests/test_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/test_lister.py @@ -0,0 +1,41 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.lister.conda.lister import CondaLister + + +def test_conda_lister_free_channel(datadir, requests_mock_datadir, swh_scheduler): + lister = CondaLister( + scheduler=swh_scheduler, channel="free", archs=["linux-64", "osx-64", "win-64"] + ) + res = lister.run() + + assert res.pages == 3 + assert res.origins == 14 + + +def test_conda_lister_msys2_channel(datadir, requests_mock_datadir, swh_scheduler): + lister = CondaLister( + scheduler=swh_scheduler, channel="msys2", archs=["win-64", "win-32"] + ) + res = lister.run() + + assert res.pages == 2 + assert res.origins == 10 + + +def test_conda_lister_conda_forge_channel( + datadir, requests_mock_datadir, swh_scheduler +): + lister = CondaLister( + scheduler=swh_scheduler, + url="https://conda.anaconda.org", + channel="conda-forge", + archs=["linux-64"], + ) + res = lister.run() + + assert res.pages == 1 + assert res.origins == 6 diff --git a/swh/lister/conda/tests/test_tasks.py b/swh/lister/conda/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/conda/tests/test_tasks.py @@ -0,0 +1,31 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.lister.pattern import ListerStats + + +def test_conda_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.conda.tasks.ping") + assert res + res.wait() + assert res.successful() + assert res.result == "OK" + + +def test_conda_lister(swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker): + # setup the mocked CondaLister + lister = mocker.patch("swh.lister.conda.tasks.CondaLister") + lister.from_configfile.return_value = lister + stats = ListerStats(pages=42, origins=42) + lister.run.return_value = stats + + res = swh_scheduler_celery_app.send_task("swh.lister.conda.tasks.CondaListerTask") + assert res + res.wait() + assert res.successful() + assert res.result == stats.dict() + + lister.from_configfile.assert_called_once_with() + lister.run.assert_called_once_with()