diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register + lister.json=swh.lister.json:register ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/lister/json/__init__.py b/swh/lister/json/__init__.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/__init__.py @@ -0,0 +1,13 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def register(): + from .models import JSONModel + from .lister import JSONLister + + return {'models': [JSONModel], + 'lister': JSONLister, + 'task_modules': ['%s.tasks' % __name__], + } diff --git a/swh/lister/json/lister.py b/swh/lister/json/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/lister.py @@ -0,0 +1,71 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import json +import requests + +from .models import JSONModel + +from swh.scheduler import utils +from swh.lister.core.simple_lister import SimpleLister + + +class JSONLister(SimpleLister): + MODEL = JSONModel + LISTER_NAME = 'json' + instance = 'json' + + def __init__(self, override_config=None): + self.url = "https://sources.nixos.org/sources.json" + super().__init__(override_config=override_config) + + def task_dict(self, origin_type, origin_url, **kwargs): + """ + Return task format dict + + This is overridden from the lister_base as more information is + needed for the ingestion task creation. + """ + return utils.create_task_dict( + 'load-tar', kwargs.get('policy', 'oneshot'), + kwargs.get('name'), + origin_url, tarballs=[{'archive': origin_url, 'date': 0}]) + + def safely_issue_request(self, identifier): + ''' + Make network request to download the JSON file. + + Args: + identifier: resource identifier (unused) + Returns: + Server response + ''' + response = requests.get(self.url, + allow_redirects=True) + # TODO: support gzip content as well + return json.loads(response.content.decode('utf-8')) + + def list_packages(self, response): + """List packages from the response + """ + return [r for r in response if r["source"]["type"] == "url"] + + def get_model_from_repo(self, source): + """Transform from source representation to model + """ + # The integrity attribute is a hash of the content in the SRI format + # See https://www.w3.org/TR/SRI + if 'integrity' in source['source']: + uid = source['source']['integrity'] + else: + uid = source['source']['url'] + + return { + 'uid': uid, + 'name': source['name'], + 'full_name': source['name'], + 'html_url': source['source']['url'], + 'origin_url': source['source']['url'], + 'origin_type': 'tar', + } diff --git a/swh/lister/json/models.py b/swh/lister/json/models.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/models.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from sqlalchemy import Column, String, Integer + +from ..core.models import ModelBase + + +class JSONModel(ModelBase): + """a JSON packages list representation + + """ + __tablename__ = 'json' + + uid = Column(String, primary_key=True) diff --git a/swh/lister/json/tasks.py b/swh/lister/json/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/tasks.py @@ -0,0 +1,17 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.scheduler.celery_backend.config import app + +from .lister import JSONLister + + +@app.task(name=__name__ + '.JSONListerTask') +def json_lister(**lister_args): + JSONLister(**lister_args).run() + + +@app.task(name=__name__ + '.ping') +def ping(): + return 'OK' diff --git a/swh/lister/json/tests/__init__.py b/swh/lister/json/tests/__init__.py new file mode 100644 diff --git a/swh/lister/json/tests/conftest.py b/swh/lister/json/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/tests/conftest.py @@ -0,0 +1,19 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from swh.lister.core.tests.conftest import * # noqa + +@pytest.fixture +def lister_json(swh_listers): + lister = swh_listers['json'] + + # Amend the credentials + lister.config = { + 'url': "https://nixos.org/sources.json" + } + + return lister diff --git a/swh/lister/json/tests/data/sources.nixos.org/sources.json b/swh/lister/json/tests/data/sources.nixos.org/sources.json new file mode 100644 --- /dev/null +++ b/swh/lister/json/tests/data/sources.nixos.org/sources.json @@ -0,0 +1,762 @@ +[ + { + "name": "hello-2.10.tar.gz", + "source": { + "hash": "0ssi1wpaf7plaswqqjwigppsg5fyh99vdlb9kzl7c9lng89ndq1i", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org//hello/hello-2.10.tar.gz", + "integrity": "sha256-MeBmE3qWJnbon2nRtlOC3pWn732RS4y5VvQepy4PUWs=" + } + }, + { + "name": "CVE-2019-5435.patch", + "source": { + "hash": "00w12yhq8q260n91i1xrynz3vn4w3lypgl19cm893s35pbvg7y17", + "hashAlgo": "sha256", + "type": "url", + "url": "https://github.com/curl/curl/commit/5fc28510a4664f4.patch", + "integrity": "sha256-J/jz9rpl6JFQZSnQdz0dnNg9vvW5hxiSBUZghKEXgQM=" + } + }, + { + "name": "curl-7.64.0.tar.bz2", + "source": { + "hash": "1szj9ia1snbfqzfcsk6hx1j7jhbqsy0f9k5d7x9xiy8w5lfblwym", + "hashAlgo": "sha256", + "type": "url", + "url": "https://curl.haxx.se/download/curl-7.64.0.tar.bz2", + "integrity": "sha256-1XO6HC0c+dhTP63M5IDXeEF5ZOjQTM3cx25ZHVRM8us=" + } + }, + { + "name": "gzip-1.10.tar.xz", + "source": { + "hash": "1h6p374d3j8d4cdfydzls021xa2yby8myc0h8d6m8bc7k6ncq9c4", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/gzip/gzip-1.10.tar.xz", + "integrity": "sha256-hCXMrJmHLVRNQxAwX5FfXqgeBND0N+8aIw3J0cgZ18A=" + } + }, + { + "name": "bzip2-1.0.6.0.1.tar.gz", + "source": { + "hash": "0b5b5p8c7bslc6fslcr1nj9136412v3qcvbg6yxi9argq9g72v8c", + "hashAlgo": "sha256", + "type": "url", + "url": "http://ftp.uni-kl.de/pub/linux/suse/people/sbrabec/bzip2/tarballs/bzip2-1.0.6.0.1.tar.gz", + "integrity": "sha256-DG1xXsIvqxS7N29thscWgZgRkrQhM6qdYVSvw9Atqyw=" + } + }, + { + "name": "xz-5.2.4.tar.bz2", + "source": { + "hash": "1gxpayfagb4v7xfhs2w6h7k56c6hwwav1rk48bj8hggljlmgs4rk", + "hashAlgo": "sha256", + "type": "url", + "url": "https://tukaani.org/xz/xz-5.2.4.tar.bz2", + "integrity": "sha256-MxP9KpX0PYjkQmTmsBXn0DBT5oGGCw1dP5usp5xXt78=" + } + }, + { + "name": "bash44-001", + "source": { + "hash": "03vzy7qwjdd5qvl3ydg99naazas2qmyd0yhnrflgjbbm64axja1y", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-001", + "integrity": "sha256-PijZFTF1LfmoyxZ60HzFQquvlE3pNT/oxqU1yfHxfw8=" + } + }, + { + "name": "bash44-002", + "source": { + "hash": "0lrwq6vyqism3yqv9s7kzaf3dsl4q5w9r5svcqz279qp7qca083h", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-002", + "integrity": "sha256-cCCgGD4XpyM+ZluXnHjBhOo2nPrz6LSxH1VH7LfBPFM=" + } + }, + { + "name": "bash44-003", + "source": { + "hash": "1chqww2rj6g42b8s60q5zlzy0jzp684jkpsbrbfy1vzxja8mmpsi", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-003", + "integrity": "sha256-Ud9akZL97+DdykvfKQky90vgP/0FA6PREuQZmQXnGLI=" + } + }, + { + "name": "bash44-004", + "source": { + "hash": "1cy8abf96hkrjhw921ndr0shlcnc52bg45rn6xri4v5clhq0l25d", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-004", + "integrity": "sha256-rQgKMKSsbBJzNzYX8pYozDIKNcjNBpE4lHlCk9xSyLM=" + } + }, + { + "name": "bash44-005", + "source": { + "hash": "0a8515kyk4zsgmvlqvlganjfr7pq0j6kzpr4d6xx02kpbdr4n7i2", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-005", + "integrity": "sha256-Ih5Lclt3CtC7aSTfP40E+J7spFWPbkx3ffqT6WcJBSk=" + } + }, + { + "name": "bash44-006", + "source": { + "hash": "1f24wgqngmj2mrj9yibwvc2zvlmn5xi53mnw777g3l40c4m2x3ka", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-006", + "integrity": "sha256-ao4uKmGA0PHOOdzWUWIvttL9Bdt8RZ9krkLWZ/HjRLg=" + } + }, + { + "name": "bash44-007", + "source": { + "hash": "1bzdsnqaf05gdbqpsixhan8vygjxpcxlz1dd8d9f5jdznw3wq76y", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-007", + "integrity": "sha256-3hzMB7e/yeJSQ62FTzu7XT6/kVWwR33xaq8Ap7DV7a8=" + } + }, + { + "name": "bash44-008", + "source": { + "hash": "1firw915mjm03hbbw9a70ch3cpgrgnvqjpllgdnn6csr8q04f546", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-008", + "integrity": "sha256-hhRHAEZZM2Nte5Reibd9+V02IANHJb4WHKDKWkLiObo=" + } + }, + { + "name": "bash44-009", + "source": { + "hash": "0g1l56kvw61rpw7dqa9fcl9llkl693h73g631hrhxlm030ddssqb", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-009", + "integrity": "sha256-C2vdGhig0g4zDMO8ceBIhk5KE2UuKdwOvzkYvqcpNDw=" + } + }, + { + "name": "bash44-010", + "source": { + "hash": "01lfhrkdsdkdz8ypzapr614ras23x7ckjnr60aa5bzkaqprccrc4", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-010", + "integrity": "sha256-hGXG8sVq/lWUAiZbOdnpQ2iVSTD5qn89+m023WaGjgY=" + } + }, + { + "name": "bash44-011", + "source": { + "hash": "038p7mhnq9m65g505hi3827jkf9f35nd1cy00w8mwafpyxp44mnx", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-011", + "integrity": "sha256-3VZCbvfXKV4RB8Cz0GwZLrkpj0AjwgLKK6YmbGE9Fw0=" + } + }, + { + "name": "bash44-012", + "source": { + "hash": "0gh6lbb1rwpk44pvbamm6vzdfi50xnwkqd9v7s8cjwk3pz973hps", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-012", + "integrity": "sha256-+sJx0r9jcsmQPjs1PLntoETX/ja1qrUvIfPyHNaiBj4=" + } + }, + { + "name": "bash44-013", + "source": { + "hash": "1djkx0w9v62q78gz3jsvamj1jq53i6hbfrfhhsw86ihwpjnfy98v", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-013", + "integrity": "sha256-GyXvrLwcRoO4htBlt6CJo2AZZFVby/EfOliYnTjoU7Y=" + } + }, + { + "name": "bash44-014", + "source": { + "hash": "0z5ikcq9zyxw79d0z36r5p0mspnb5piavbv03jmlan1wnknmrxx7", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-014", + "integrity": "sha256-p/dc7bQ8WEWrHGCvreIty15dwS3ZjA9aOrz7nzCbsXw=" + } + }, + { + "name": "bash44-015", + "source": { + "hash": "09n307fi1j257abhm295k6ksmnzw47ka2zhnr0i5lbdnpvn04xnk", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-015", + "integrity": "sha256-03YC7L62LVoiyBZ+oeYh/Nuqp5kliQqXOkXIEN0BwyY=" + } + }, + { + "name": "bash44-016", + "source": { + "hash": "1cgi1y6mifm8hsgv4avj5ih76535js3qba1sqwbfvp7si76927sh", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-016", + "integrity": "sha256-UB+RzIn63O0WxzqohYeWZRRzYCxyK7Kfhqi6WI0P8bE=" + } + }, + { + "name": "bash44-017", + "source": { + "hash": "0w6jpj2giakji1ir83rpkx1y7n7xqppah3j748m6dm38hywr0gvp", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-017", + "integrity": "sha256-dz+QuYdo1GYqIkcOqO7F/djjQ583D5RjiHKq+IS80nA=" + } + }, + { + "name": "bash44-018", + "source": { + "hash": "1k58h4wxbsg7r4rwhrvzx5hfbapba2nxjysbhh6qp6ki5ys99i2v", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-018", + "integrity": "sha256-W8SUtC9xmosNhEt72a1Q66rlYOl/Z8gzyefp1TmBqMw=" + } + }, + { + "name": "bash44-019", + "source": { + "hash": "07n1i5610lbs672x1s8g82qn3qfj06s0ip3z80sri0g8vxp0s5r7", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-019", + "integrity": "sha256-JxcNbt/ogZg1QH/cCLQB0uFhsUAP6dDFMXpREEyJwR4=" + } + }, + { + "name": "bash44-020", + "source": { + "hash": "0b2jk5n1af1vh590qfc52hv65mafb4vl1xv26s8j5a3byb5y4h0q", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-020", + "integrity": "sha256-GEDiy/JrqCKRNmL3QDdZTtViNhSFOQxSgTs4FWyZUiw=" + } + }, + { + "name": "bash44-021", + "source": { + "hash": "1hblcd2xmqqlp0idnavw66570n7m0yv5rbbr873c2gkn982mk3xx", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-021", + "integrity": "sha256-vY9ZBUp2PsHGQXmtXLYH9VhwijF8K9siuBTj2kVjdME=" + } + }, + { + "name": "bash44-022", + "source": { + "hash": "0yfbjzr79vzjs2hyi5m8iy2b38fq7vikdfa4zqdvjsp36q4iycs5", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-022", + "integrity": "sha256-RTMfCTbjarkb/kS5NuM+2KGxhI+oluih0PLvdPKXy3k=" + } + }, + { + "name": "bash44-023", + "source": { + "hash": "1dlism6qdx60nvzj0v7ndr7lfahl4a8zmzckp13hqgdx7xpj7v2g", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-023", + "integrity": "sha256-T+wjbz+9PQxHuJP9+pEiFCpHT272bCD/tsD0hk3VkbY=" + } + }, + { + "name": "bash-4.4.tar.gz", + "source": { + "hash": "1jyz6snd63xjn6skk7za6psgidsd53k05cr3lksqybi0q6936syq", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bash/bash-4.4.tar.gz", + "integrity": "sha256-2GszksEgLo/1pCOzAuYoTbf49DXqnzm1sbIP06w238s=" + } + }, + { + "name": "binutils-2.31.1.tar.bz2", + "source": { + "hash": "1l34hn1zkmhr1wcrgf0d4z7r3najxnw3cx2y2fk7v55zjlk3ik7z", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/binutils/binutils-2.31.1.tar.bz2", + "integrity": "sha256-/8w4JpW/lH2mE150NrjtUtmRzycNuJcZDxnW+YOFZNA=" + } + }, + { + "name": "coreutils-8.30.tar.xz", + "source": { + "hash": "0mxhw43d4wpqmvg0l4znk1vm10fy92biyh90lzdnqjcic2lb6cg8", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/coreutils/coreutils-8.30.tar.xz", + "integrity": "sha256-6DGzqGCRSWzbpyBBH5dI3oFQd5j2Ewrervhy0gbhsFc=" + } + }, + { + "name": "diffutils-3.7.tar.xz", + "source": { + "hash": "09isrg0isjinv8c535nxsi1s86wfdfzml80dbw41dj9x3hiad9xk", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/diffutils/diffutils-3.7.tar.xz", + "integrity": "sha256-s6emIhw9yRYIXw0gWr9rjhukQ9TdllEY2jZKHcHLOiY=" + } + }, + { + "name": "findutils-4.6.0.tar.gz", + "source": { + "hash": "178nn4dl7wbcw499czikirnkniwnx36argdnqgz4ik9i6zvwkm6y", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/findutils/findutils-4.6.0.tar.gz", + "integrity": "sha256-3tTJ9zcxzUj+w7a9rMzolkc7bY4zfpYS4WzxQxuxFp0=" + } + }, + { + "name": "gawk-4.2.1.tar.xz", + "source": { + "hash": "0lam2zf3n7ak4pig8w46lhx9hzx50kj2v2yj1616mm26wy2rf4fi", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/gawk/gawk-4.2.1.tar.xz", + "integrity": "sha256-0RGXhedG1GqCCdKLLeQEpX+YOqSGcPTiJVMdO9wXVVE=" + } + }, + { + "name": "make-4.2.1.tar.bz2", + "source": { + "hash": "12f5zzyq2w56g95nni65hc0g5p7154033y2f3qmjvd016szn5qnn", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/make/make-4.2.1.tar.bz2", + "integrity": "sha256-1uJivzYBtC0rHk74MQAp4dzyAIPFRGtLeqZwgf3/xYk=" + } + }, + { + "name": "sed-4.7.tar.xz", + "source": { + "hash": "0smxcx66vx29djzb542nxcynl7qnzxqa5032ibazi7x2s267d198", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/sed/sed-4.7.tar.xz", + "integrity": "sha256-KIV2jNCin/jVimKAonD/Fh9qPetWkLK+bEn0bUxnvWo=" + } + }, + { + "name": "tar-1.31.tar.xz", + "source": { + "hash": "1h9dxhjhz1jnyhmh6jfhqw1g1sxqbg3cd32vpwg7x2xxxqffzwrp", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/tar/tar-1.31.tar.xz", + "integrity": "sha256-N/PvHO69i34ev1uMxsZbuOvwAsfQSQMr9FaGDyXsLcE=" + } + }, + { + "name": "grep-3.3.tar.xz", + "source": { + "hash": "055mqp6vrd0brkygmygb2673qwz409a7kyp1mzbfy6cn94f58q5r", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/grep/grep-3.3.tar.xz", + "integrity": "sha256-uWBUHEmWGe/Wr+H6eVQC5HM8jhHr+fr8zAu0vM3FtRQ=" + } + }, + { + "name": "Allow_input_files_to_be_missing_for_ed-style_patches.patch", + "source": { + "hash": "0iw0lk0yhnhvfjzal48ij6zdr92mgb84jq7fwryy1hdhi47hhq64", + "hashAlgo": "sha256", + "type": "url", + "url": "https://git.savannah.gnu.org/cgit/patch.git/patch/?id=b5a91a01e5d0897facdd0f49d64b76b0f02b43e1", + "integrity": "sha256-xGAID4mwweB95u5gSdB6VaTcvpEREaq+dBta6MGkgEc=" + } + }, + { + "name": "Fix_arbitrary_command_execution_in_ed-style_patches.patch", + "source": { + "hash": "1bpy16n3hm5nv9xkrn6c4wglzsdzj3ss1biq16w9kfv48p4hx2vg", + "hashAlgo": "sha256", + "type": "url", + "url": "https://git.savannah.gnu.org/cgit/patch.git/patch/?id=123eaff0d5d1aebe128295959435b9ca5909c26d", + "integrity": "sha256-b4sOyUVku5m4CTiuoPWQv+lPHyfM2Dx72rZUOKwJ/q4=" + } + }, + { + "name": "patch-2.7.6.tar.xz", + "source": { + "hash": "1zfqy4rdcy279vwn2z1kbv19dcfw25d2aqy9nzvdkq5bjzd0nqdc", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/patch/patch-2.7.6.tar.xz", + "integrity": "sha256-rGEL2per4Nn2t8ljJVoR3LGWwl4zfGH5Tkd41jLx2P0=" + } + }, + { + "name": "patchelf-0.9.tar.bz2", + "source": { + "hash": "a0f65c1ba148890e9f2f7823f4bedf7ecad5417772f64f994004f59a39014f83", + "hashAlgo": "sha256", + "type": "url", + "url": "https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2", + "integrity": "sha256-oPZcG6FIiQ6fL3gj9L7ffsrVQXdy9k+ZQAT1mjkBT4M=" + } + }, + { + "name": "ed-1.15.tar.lz", + "source": { + "hash": "0x6ivy5k0d7dy5z9g8q8nipr89m4qbk2ink2898qq43smp08ji5d", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/ed/ed-1.15.tar.lz", + "integrity": "sha256-rUSJwK16EIxRQmLaKObCpCaUb7QIo5d+8e00MIvf0XQ=" + } + }, + { + "name": "?id=14adc898a36948267bfe5c63b399996879e94c98", + "source": { + "hash": "0gja54fz79a9ma6b4mprnjxq77l5yg2z9xknlwhmkcrfnam02qxp", + "hashAlgo": "sha256", + "type": "url", + "url": "https://git.savannah.nongnu.org/cgit/attr.git/patch/?id=14adc898a36948267bfe5c63b399996879e94c98", + "integrity": "sha256-t2MBqrIus1khp3b29MXzhZ6Du7T5VrKMqkml8x0pSj4=" + } + }, + { + "name": "attr-2.4.48.tar.gz", + "source": { + "hash": "1rr4adzwax4bzr2c00f06zcsljv5y6p9wymz1g89ww7cb2rp5bay", + "hashAlgo": "sha256", + "type": "url", + "url": "http://mirror.easyname.at/nongnu/attr/attr-2.4.48.tar.gz", + "integrity": "sha256-Xq1ys1jscJ7QC796nq7xZUuq2TfAAcBE/ot0xX9TJOc=" + } + }, + { + "name": "acl-2.2.53.tar.gz", + "source": { + "hash": "1ir6my3w74s6nfbgbqgzj6w570sn0qjf3524zx8xh67lqrjrigh6", + "hashAlgo": "sha256", + "type": "url", + "url": "http://mirror.easyname.at/nongnu/acl/acl-2.2.53.tar.gz", + "integrity": "sha256-Br6YZcb0GNhR/0SU4SQGVoNTuJH/4fWWs0aTw4evJsc=" + } + }, + { + "name": "zlib-1.2.11.tar.gz", + "source": { + "hash": "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1", + "hashAlgo": "sha256", + "type": "url", + "url": "https://www.zlib.net/fossils/zlib-1.2.11.tar.gz", + "integrity": "sha256-w+Xp/dUATctUL+2l7k8P8HRGKLr47S3V1m+MoRl8saE=" + } + }, + { + "name": "pcre-8.42.tar.bz2", + "source": { + "hash": "00ckpzlgyr16bnqx8fawa3afjgqxw5yxgs2l081vw23qi1y4pl1c", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftp.pcre.org/pub/pcre/pcre-8.42.tar.bz2", + "integrity": "sha256-LNBLfIh4CL4DAlTo133hHT/p1FBcOdSxXSZk/+i/kwE=" + } + }, + { + "name": "locale-C.diff", + "source": { + "hash": "0irj60hs2i91ilwg5w7sqrxb695c93xg0ik7yhhq9irprd7fidn4", + "hashAlgo": "sha256", + "type": "url", + "url": "https://salsa.debian.org/glibc-team/glibc/raw/49767c9f7de4828220b691b29de0baf60d8a54ec/debian/patches/localedata/locale-C.diff", + "integrity": "sha256-xLboTss3x4Qh9GdG8PpIrCSzesb68PI4jSFFoSEwMkc=" + } + }, + { + "name": "glibc-2.27.tar.xz", + "source": { + "hash": "0wpwq7gsm7sd6ysidv0z575ckqdg13cr2njyfgrbgh4f65adwwji", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/glibc/glibc-2.27.tar.xz", + "integrity": "sha256-UXLeVDGOwLfyc15akdkIr+HJyikf7Ba1N02fqt/B/HI=" + } + }, + { + "name": "linux-4.19.16.tar.xz", + "source": { + "hash": "1pqvn6dsh0xhdpawz4ag27vkw1abvb6sn3869i4fbrz33ww8i86q", + "hashAlgo": "sha256", + "type": "url", + "url": "http://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.16.tar.xz", + "integrity": "sha256-2KCIOB/j5+VITAYNq83aSwU+9xFPkc/VbbADqJuxG98=" + } + }, + { + "name": "gcc-7.4.0.tar.xz", + "source": { + "hash": "0lgy170b0pp60j9cczqkmaqyjjb584vfamj4c30swd7k0j6y5pgd", + "hashAlgo": "sha256", + "type": "url", + "url": "https://bigsearcher.com/mirrors/gcc/releases/gcc-7.4.0/gcc-7.4.0.tar.xz", + "integrity": "sha256-7d3ijQTzNK7BYERW5TZBZUnpsaoTf8aSBOZesMAJ/lE=" + } + }, + { + "name": "bootstrap-tools.tar.xz", + "source": { + "hash": "abe3f0727dd771a60b7922892d308da1bc7b082afc13440880862f0c8823c09f", + "hashAlgo": "sha256", + "type": "url", + "url": "http://tarballs.nixos.org/stdenv-linux/x86_64/4907fc9e8d0d82b28b3c56e3a478a2882f1d700f/bootstrap-tools.tar.xz", + "integrity": "sha256-q+Pwcn3XcaYLeSKJLTCNobx7CCr8E0QIgIYvDIgjwJ8=" + } + }, + { + "name": "pkg-config-0.29.2.tar.gz", + "source": { + "hash": "14fmwzki1rlz8bs2p810lk6jqdxsk966d8drgsjmi54cd00rrikg", + "hashAlgo": "sha256", + "type": "url", + "url": "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz", + "integrity": "sha256-b8acAWiMlFilfrmhZkyaujcszaQgoCv0Qp/mEOfn1ZE=" + } + }, + { + "name": "0001-Fix-missing-build-dependency-for-pods.patch", + "source": { + "hash": "1bb4mldfp8kq1scv480wm64n2jdsqa3ar46cjp1mjpby8h5dr2r0", + "hashAlgo": "sha256", + "type": "url", + "url": "https://rt.perl.org/Public/Ticket/Attachment/1502646/807252/0001-Fix-missing-build-dependency-for-pods.patch", + "integrity": "sha256-IIvcCkR+XVnDlcyQrIbCuklhiakcILKZDnii6xqtZK0=" + } + }, + { + "name": "perl-5.28.1.tar.gz", + "source": { + "hash": "0iy3as4hnbjfyws4in3j9d6zhhjxgl5m95i5n9jy2bnzcpz8bgry", + "hashAlgo": "sha256", + "type": "url", + "url": "https://cpan.metacpan.org/src/5.0/perl-5.28.1.tar.gz", + "integrity": "sha256-Pr+F/mXfLuFlsiWWVAt9XUL4TUty2Eg0904uC4lWw0c=" + } + }, + { + "name": "openssl-1.0.2t.tar.gz", + "source": { + "hash": "1g67ra0ph7gpz6fgvv1i96d792jmd6ymci5kk53vbikszr74djql", + "hashAlgo": "sha256", + "type": "url", + "url": "https://www.openssl.org/source/openssl-1.0.2t.tar.gz", + "integrity": "sha256-FMtGTv56xrVHmbNEVr1pVYp0mkkx7P2c+fcdeIHKx7w=" + } + }, + { + "name": "nghttp2-1.36.0.tar.bz2", + "source": { + "hash": "1sqwwwhlrbp3wcikf9v07k5j4xvflwy297lq4cg92qj087bk99qn", + "hashAlgo": "sha256", + "type": "url", + "url": "https://github.com/nghttp2/nghttp2/releases/download/v1.36.0/nghttp2-1.36.0.tar.bz2", + "integrity": "sha256-Fqc010FAYpEeI5ieJDynbnciyzxgJzcj4+OuTCHnHOs=" + } + }, + { + "name": "krb5-1.15.2.tar.gz", + "source": { + "hash": "0zn8s7anb10hw3nzwjz7vg10fgmmgvwnibn2zrn3nppjxn9f6f8n", + "hashAlgo": "sha256", + "type": "url", + "url": "http://web.mit.edu/kerberos/dist/krb5/1.15/krb5-1.15.2.tar.gz", + "integrity": "sha256-Fjnjku3yXjts/sKuaPl+tT4HwtvnS/7t4BCEZdXRyH4=" + } + }, + { + "name": "libssh2-1.9.0.tar.gz", + "source": { + "hash": "1zfsz9nldakfz61d2j70pk29zlmj7w2vv46s9l3x2prhcgaqpyym", + "hashAlgo": "sha256", + "type": "url", + "url": "https://www.libssh2.org/download/libssh2-1.9.0.tar.gz", + "integrity": "sha256-1fuL1WMwX9EHTdqQvQU/stKfxLzgSNGC+W6qRm362v0=" + } + }, + { + "name": "bison-3.3.2.tar.gz", + "source": { + "hash": "1lfvns1f6xamz9905fakzkxh4zmk1wa9w33vxcq78fc5841ivnhg", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/bison/bison-3.3.2.tar.gz", + "integrity": "sha256-D9odA0GFOXQw63sMnhQPs34C+/xTuQJS+lV144K229E=" + } + }, + { + "name": "gmp-6.1.2.tar.bz2", + "source": { + "hash": "1clg7pbpk6qwxj5b2mw0pghzawp2qlm3jf9gdd8i6fl6yh2bnxaj", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/gmp/gmp-6.1.2.tar.bz2", + "integrity": "sha256-UnW7BPSGOhNRay85OSrF4nL14buAV7GK7Bybedc9j7I=" + } + }, + { + "name": "lzip-1.21.tar.gz", + "source": { + "hash": "12qdcw5k1cx77brv9yxi1h4dzwibhfmdpigrj43nfk8nscwm12z4", + "hashAlgo": "sha256", + "type": "url", + "url": "http://mirror.easyname.at/nongnu/lzip/lzip-1.21.tar.gz", + "integrity": "sha256-5ItQOdMWTWcHkfnF26qDK/LfCAyx+7TzOqezMAtnDYs=" + } + }, + { + "name": "gettext-0.19.8.1.tar.gz", + "source": { + "hash": "0hsw28f9q9xaggjlsdp2qmbp2rbd1mp0njzan2ld9kiqwkq2m57z", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/gettext/gettext-0.19.8.1.tar.gz", + "integrity": "sha256-/5Qq8OQ4ztSosOpLC24NbWVxV8XiNk3le6onnBwSXEM=" + } + }, + { + "name": "mpfr-4.0.2.tar.xz", + "source": { + "hash": "12m3amcavhpqygc499s3fzqlb8f2j2rr7fkqsm10xbjfc04fffqx", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.2.tar.xz", + "integrity": "sha256-HTvnCGBOrg5C1Xi6k7OQwqFF8XdDp0TY8/jCrVhVo4o=" + } + }, + { + "name": "mpc-1.1.0.tar.gz", + "source": { + "hash": "0biwnhjm3rx3hc0rfpvyniky4lpzsvdcwhmcn7f0h4iw2hwcb1b9", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/mpc/mpc-1.1.0.tar.gz", + "integrity": "sha256-aYXFOBQ8EgjcsaxCztrW/1LiZ7R+X5cBg6PnUSW0PC4=" + } + }, + { + "name": "libelf-0.8.13.tar.gz", + "source": { + "hash": "0vf7s9dwk2xkmhb79aigqm0x0yfbw1j0b9ksm51207qwr179n6jr", + "hashAlgo": "sha256", + "type": "url", + "url": "https://fossies.org/linux/misc/old/libelf-0.8.13.tar.gz", + "integrity": "sha256-WRqbTsgcHyBCqXqmBWTgy3nQQcUvqnQWrLOLyVvSx20=" + } + }, + { + "name": "isl-0.17.1.tar.xz", + "source": { + "hash": "be152e5c816b477594f4c6194b5666d8129f3a27702756ae9ff60346a8731647", + "hashAlgo": "sha256", + "type": "url", + "url": "http://isl.gforge.inria.fr/isl-0.17.1.tar.xz", + "integrity": "sha256-vhUuXIFrR3WU9MYZS1Zm2BKfOidwJ1aun/YDRqhzFkc=" + } + }, + { + "name": "texinfo-6.5.tar.xz", + "source": { + "hash": "0qjzvbvnv9003xdrcpi3jp7y68j4hq2ciw9frh2hghh698zlnxvp", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/texinfo/texinfo-6.5.tar.xz", + "integrity": "sha256-d3dLP0oGwgcFzC7xyASGRCLjz5UjXpZbHwCkbffaX2I=" + } + }, + { + "name": "which-2.21.tar.gz", + "source": { + "hash": "1bgafvy3ypbhhfznwjv1lxmd6mci3x1byilnnkc7gcr486wlb8pl", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/which/which-2.21.tar.gz", + "integrity": "sha256-9KJFuUEks3fYtJZGv0IfkVXTaqdhS26/g3BdP/x26q0=" + } + }, + { + "name": "libev-4.25.tar.gz", + "source": { + "hash": "1295q0lkkbrlpd5dl5i48bh1rm8mjzh9y795jlvjz3bp4wf7wxbq", + "hashAlgo": "sha256", + "type": "url", + "url": "http://dist.schmorp.de/libev/Attic/libev-4.25.tar.gz", + "integrity": "sha256-eHV+HCd3jS83lSUdn+CXFdUc4EIkFtpKuzSvOSnAJYk=" + } + }, + { + "name": "c-ares-1.15.0.tar.gz", + "source": { + "hash": "0lk8knip4xk6qzksdkn7085mmgm4ixfczdyyjw656c193y3rgnvc", + "hashAlgo": "sha256", + "type": "url", + "url": "https://c-ares.haxx.se/download/c-ares-1.15.0.tar.gz", + "integrity": "sha256-bNuXhx8pMFMMl963z1yPpL5aCwLHzqbnx2Z2cqOdaFI=" + } + }, + { + "name": "Make-build-reproducible.patch", + "source": { + "hash": "0wnvbjfrbk7rghd032z684l7vk7mhy3bd41zvhkrhgp3cd5id0bm", + "hashAlgo": "sha256", + "type": "url", + "url": "https://salsa.debian.org/debian/keyutils/raw/4cecffcb8e2a2aa4ef41777ed40e4e4bcfb2e5bf/debian/patches/Make-build-reproducible.patch", + "integrity": "sha256-dYEWS2PjPpgn3D+QtoaH9cx9KEHmiwEafPnMlZ1c23I=" + } + }, + { + "name": "keyutils-1.6.tar.bz2", + "source": { + "hash": "05bi5ja6f3h3kdi7p9dihlqlfrsmi1wh1r2bdgxc0180xh6g5bnk", + "hashAlgo": "sha256", + "type": "url", + "url": "https://people.redhat.com/dhowells/keyutils/keyutils-1.6.tar.bz2", + "integrity": "sha256-067yDOwABcD6a0vkAHmIVWdHMYWxpXtimwMOZ5QscRU=" + } + }, + { + "name": "m4-1.4.18.tar.bz2", + "source": { + "hash": "1xkwwq0sgv05cla0g0a01yzhk0wpsn9y40w9kh9miiiv0imxfh36", + "hashAlgo": "sha256", + "type": "url", + "url": "https://ftpmirror.gnu.org/m4/m4-1.4.18.tar.bz2", + "integrity": "sha256-ZkDXawQ7xlgTnIkD4pPVl4MJvw9AgQcUZQXspwHmfPY=" + } + } +] diff --git a/swh/lister/json/tests/test_lister.py b/swh/lister/json/tests/test_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/tests/test_lister.py @@ -0,0 +1,25 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_lister_no_page_check_results(lister_json, requests_mock_datadir): + lister = lister_json +# lister.url = "http://sources.nixos.org/packages.json" + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-tar') + assert len(r) == 76 + + for row in r: + print(row) + assert row['type'] == 'load-tar' + # arguments check + args = row['arguments']['args'] + assert len(args) == 2 + + # kwargs + kwargs = row['arguments']['kwargs'] + assert list(kwargs.keys()) == ['tarballs'] + + assert row['policy'] == 'oneshot' diff --git a/swh/lister/json/tests/test_tasks.py b/swh/lister/json/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/json/tests/test_tasks.py @@ -0,0 +1,27 @@ +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.json.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.json.tasks.JSONLister') +def test_lister(lister, swh_app, celery_session_worker): + # setup the mocked JSONLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.json.tasks.JSONListerTask') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with()