diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -65,6 +65,7 @@ lister.packagist=swh.lister.packagist:register lister.phabricator=swh.lister.phabricator:register lister.pypi=swh.lister.pypi:register + lister.functional_package=swh.lister.functional_package:register ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/lister/functional_package/__init__.py b/swh/lister/functional_package/__init__.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/__init__.py @@ -0,0 +1,13 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def register(): + from .models import FunctionalPackageModel + from .lister import FunctionalPackageLister + + return {'models': [FunctionalPackageModel], + 'lister': FunctionalPackageLister, + 'task_modules': ['%s.tasks' % __name__], + } diff --git a/swh/lister/functional_package/lister.py b/swh/lister/functional_package/lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/lister.py @@ -0,0 +1,69 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import json +import requests + +from .models import FunctionalPackageModel + +from swh.scheduler import utils +from swh.lister.core.simple_lister import SimpleLister + + +class FunctionalPackageLister(SimpleLister): + MODEL = FunctionalPackageModel + LISTER_NAME = 'functional_package' + instance = 'functional_package' + + def __init__(self, + url="https://tarballs.nixos.org/sources.json", + override_config=None): + self.url = url + super().__init__(override_config=override_config) + + def task_dict(self, origin_type, origin_url, **kwargs): + '''Return a task loading exactly one source. + + The JSON file exposed by NixOS and Guix lists all sources used + to build their package sets. Since this file doesn't have the notion + of packages, it only exposes sources. + ''' + return utils.create_task_dict( + 'load-tar', kwargs.get('policy', 'oneshot'), + url=origin_url, + artifacts=[{'archive': origin_url}], + identity_artifact_keys=['archive'], + retries_left=3) + + def safely_issue_request(self, identifier): + ''' + Make network request to download the JSON file. + + Args: + identifier: resource identifier (unused) + Returns: + Server response + ''' + response = requests.get(self.url, + allow_redirects=True) + # TODO: support gzip content as well + return json.loads(response.content.decode('utf-8')) + + def list_packages(self, response): + """List packages from the response + """ + return [r for r in response["sources"] if r["type"] == "url"] + + def get_model_from_repo(self, source): + """Transform from source representation to model + """ + return { + # We could use the content hash if it is provided + 'uid': source['url'], + 'name': source['url'], + 'full_name': source['url'], + 'html_url': source['url'], + 'origin_url': source['url'], + 'origin_type': 'tar', + } diff --git a/swh/lister/functional_package/models.py b/swh/lister/functional_package/models.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/models.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from sqlalchemy import Column, String + +from ..core.models import ModelBase + + +class FunctionalPackageModel(ModelBase): + """a functional package packages representation + + """ + __tablename__ = 'functional_package' + + uid = Column(String, primary_key=True) diff --git a/swh/lister/functional_package/tasks.py b/swh/lister/functional_package/tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/tasks.py @@ -0,0 +1,17 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.scheduler.celery_backend.config import app + +from .lister import FunctionalPackageLister + + +@app.task(name=__name__ + '.FunctionalPackageListerTask') +def functional_package_lister(**lister_args): + FunctionalPackageLister(**lister_args).run() + + +@app.task(name=__name__ + '.ping') +def ping(): + return 'OK' diff --git a/swh/lister/functional_package/tests/__init__.py b/swh/lister/functional_package/tests/__init__.py new file mode 100644 diff --git a/swh/lister/functional_package/tests/conftest.py b/swh/lister/functional_package/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/tests/conftest.py @@ -0,0 +1,20 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from swh.lister.core.tests.conftest import * # noqa + + +@pytest.fixture +def lister_functional_package(swh_listers): + lister = swh_listers['functional_package'] + + # Amend the credentials + lister.config = { + 'url': "https://nixos.org/sources.json" + } + + return lister diff --git a/swh/lister/functional_package/tests/data/https_tarballs.nixos.org/sources.json b/swh/lister/functional_package/tests/data/https_tarballs.nixos.org/sources.json new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/tests/data/https_tarballs.nixos.org/sources.json @@ -0,0 +1,29 @@ +{ + "sources": [ + { + "type": "url", + "url": "https://ftpmirror.gnu.org//hello/hello-2.10.tar.gz" + }, + { + "type": "url", + "url": "https://github.com/curl/curl/commit/5fc28510a4664f4.patch" + }, + { + "type": "url", + "url": "https://curl.haxx.se/download/curl-7.64.0.tar.bz2" + }, + { + "type": "url", + "url": "https://ftpmirror.gnu.org/gzip/gzip-1.10.tar.xz" + }, + { + "type": "url", + "url": "http://ftp.uni-kl.de/pub/linux/suse/people/sbrabec/bzip2/tarballs/bzip2-1.0.6.0.1.tar.gz" + }, + { + "type": "url", + "url": "https://tukaani.org/xz/xz-5.2.4.tar.bz2" + } + ], + "version": 1 +} diff --git a/swh/lister/functional_package/tests/test_lister.py b/swh/lister/functional_package/tests/test_lister.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/tests/test_lister.py @@ -0,0 +1,26 @@ +# Copyright (C) 2019 the Software Heritage developers +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +def test_lister_no_page_check_results(lister_functional_package, + requests_mock_datadir): + lister = lister_functional_package + lister.run() + + r = lister.scheduler.search_tasks(task_type='load-tar') + assert len(r) == 6 + + for row in r: + assert row['type'] == 'load-tar' + # arguments check + args = row['arguments']['args'] + assert len(args) == 0 + + # kwargs + kwargs = row['arguments']['kwargs'] + assert list(kwargs.keys()) == ['url', + 'artifacts', + 'identity_artifact_keys'] + + assert row['policy'] == 'oneshot' diff --git a/swh/lister/functional_package/tests/test_tasks.py b/swh/lister/functional_package/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/functional_package/tests/test_tasks.py @@ -0,0 +1,27 @@ +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.functional_package.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.functional_package.tasks.FunctionalPackageLister') +def test_lister(lister, swh_app, celery_session_worker): + # setup the mocked JSONLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.functional_package.tasks.FunctionalPackageListerTask') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with()