Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F9696362
D2025.id7800.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
16 KB
Subscribers
None
D2025.id7800.diff
View Options
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -65,6 +65,7 @@
lister.packagist=swh.lister.packagist:register
lister.phabricator=swh.lister.phabricator:register
lister.pypi=swh.lister.pypi:register
+ lister.functional_package=swh.lister.functional_package:register
''',
classifiers=[
"Programming Language :: Python :: 3",
diff --git a/swh/lister/functional_package/__init__.py b/swh/lister/functional_package/__init__.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/__init__.py
@@ -0,0 +1,13 @@
+# Copyright (C) 2019 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def register():
+ from .models import FunctionalPackageModel
+ from .lister import FunctionalPackageLister
+
+ return {'models': [FunctionalPackageModel],
+ 'lister': FunctionalPackageLister,
+ 'task_modules': ['%s.tasks' % __name__],
+ }
diff --git a/swh/lister/functional_package/lister.py b/swh/lister/functional_package/lister.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/lister.py
@@ -0,0 +1,64 @@
+# Copyright (C) 2019 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import json
+import requests
+
+from .models import FunctionalPackageModel
+
+from swh.scheduler import utils
+from swh.lister.core.simple_lister import SimpleLister
+
+
+class FunctionalPackageLister(SimpleLister):
+ MODEL = FunctionalPackageModel
+ LISTER_NAME = 'functional_package'
+ instance = 'functional_package'
+
+ def __init__(self, override_config=None):
+ self.url = "https://sources.nixos.org/sources.json"
+ super().__init__(override_config=override_config)
+
+ def task_dict(self, origin_type, origin_url, **kwargs):
+ """
+ Return task format dict
+
+ This is overridden from the lister_base as more information is
+ needed for the ingestion task creation.
+ """
+ return utils.create_task_dict(
+ 'load-tar', kwargs.get('policy', 'oneshot'),
+ origin_url, tarballs=[{'archive': origin_url, 'date': 0}])
+
+ def safely_issue_request(self, identifier):
+ '''
+ Make network request to download the JSON file.
+
+ Args:
+ identifier: resource identifier (unused)
+ Returns:
+ Server response
+ '''
+ response = requests.get(self.url,
+ allow_redirects=True)
+ # TODO: support gzip content as well
+ return json.loads(response.content.decode('utf-8'))
+
+ def list_packages(self, response):
+ """List packages from the response
+ """
+ return [r for r in response["sources"] if r["type"] == "url"]
+
+ def get_model_from_repo(self, source):
+ """Transform from source representation to model
+ """
+ return {
+ # We could use the content hash if it is provided
+ 'uid': source['url'],
+ 'name': source['url'],
+ 'full_name': source['url'],
+ 'html_url': source['url'],
+ 'origin_url': source['url'],
+ 'origin_type': 'tar',
+ }
diff --git a/swh/lister/functional_package/models.py b/swh/lister/functional_package/models.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/models.py
@@ -0,0 +1,16 @@
+# Copyright (C) 2019 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from sqlalchemy import Column, String
+
+from ..core.models import ModelBase
+
+
+class FunctionalPackageModel(ModelBase):
+ """a functional package packages representation
+
+ """
+ __tablename__ = 'functional_package'
+
+ uid = Column(String, primary_key=True)
diff --git a/swh/lister/functional_package/tasks.py b/swh/lister/functional_package/tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/tasks.py
@@ -0,0 +1,17 @@
+# Copyright (C) 2019 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from swh.scheduler.celery_backend.config import app
+
+from .lister import FunctionalPackageLister
+
+
+@app.task(name=__name__ + '.FunctionalPackageListerTask')
+def functional_package_lister(**lister_args):
+ FunctionalPackageLister(**lister_args).run()
+
+
+@app.task(name=__name__ + '.ping')
+def ping():
+ return 'OK'
diff --git a/swh/lister/functional_package/tests/__init__.py b/swh/lister/functional_package/tests/__init__.py
new file mode 100644
diff --git a/swh/lister/functional_package/tests/conftest.py b/swh/lister/functional_package/tests/conftest.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/tests/conftest.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+from swh.lister.core.tests.conftest import * # noqa
+
+
+@pytest.fixture
+def lister_functional_package(swh_listers):
+ lister = swh_listers['functional_package']
+
+ # Amend the credentials
+ lister.config = {
+ 'url': "https://nixos.org/sources.json"
+ }
+
+ return lister
diff --git a/swh/lister/functional_package/tests/data/sources.nixos.org/sources.json b/swh/lister/functional_package/tests/data/sources.nixos.org/sources.json
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/tests/data/sources.nixos.org/sources.json
@@ -0,0 +1,309 @@
+{
+ "sources": [
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org//hello/hello-2.10.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://github.com/curl/curl/commit/5fc28510a4664f4.patch"
+ },
+ {
+ "type": "url",
+ "url": "https://curl.haxx.se/download/curl-7.64.0.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/gzip/gzip-1.10.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "http://ftp.uni-kl.de/pub/linux/suse/people/sbrabec/bzip2/tarballs/bzip2-1.0.6.0.1.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://tukaani.org/xz/xz-5.2.4.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-001"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-002"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-003"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-004"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-005"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-006"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-007"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-008"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-009"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-010"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-011"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-012"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-013"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-014"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-015"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-016"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-017"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-018"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-019"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-020"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-021"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-022"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4-patches/bash44-023"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bash/bash-4.4.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/binutils/binutils-2.31.1.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/coreutils/coreutils-8.30.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/diffutils/diffutils-3.7.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/findutils/findutils-4.6.0.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/gawk/gawk-4.2.1.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/make/make-4.2.1.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/sed/sed-4.7.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/tar/tar-1.31.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/grep/grep-3.3.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://git.savannah.gnu.org/cgit/patch.git/patch/?id=b5a91a01e5d0897facdd0f49d64b76b0f02b43e1"
+ },
+ {
+ "type": "url",
+ "url": "https://git.savannah.gnu.org/cgit/patch.git/patch/?id=123eaff0d5d1aebe128295959435b9ca5909c26d"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/patch/patch-2.7.6.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://nixos.org/releases/patchelf/patchelf-0.9/patchelf-0.9.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/ed/ed-1.15.tar.lz"
+ },
+ {
+ "type": "url",
+ "url": "https://git.savannah.nongnu.org/cgit/attr.git/patch/?id=14adc898a36948267bfe5c63b399996879e94c98"
+ },
+ {
+ "type": "url",
+ "url": "http://mirror.easyname.at/nongnu/attr/attr-2.4.48.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "http://mirror.easyname.at/nongnu/acl/acl-2.2.53.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://www.zlib.net/fossils/zlib-1.2.11.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftp.pcre.org/pub/pcre/pcre-8.42.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://salsa.debian.org/glibc-team/glibc/raw/49767c9f7de4828220b691b29de0baf60d8a54ec/debian/patches/localedata/locale-C.diff"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/glibc/glibc-2.27.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "http://cdn.kernel.org/pub/linux/kernel/v4.x/linux-4.19.16.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://bigsearcher.com/mirrors/gcc/releases/gcc-7.4.0/gcc-7.4.0.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "http://tarballs.nixos.org/stdenv-linux/x86_64/4907fc9e8d0d82b28b3c56e3a478a2882f1d700f/bootstrap-tools.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.2.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://rt.perl.org/Public/Ticket/Attachment/1502646/807252/0001-Fix-missing-build-dependency-for-pods.patch"
+ },
+ {
+ "type": "url",
+ "url": "https://cpan.metacpan.org/src/5.0/perl-5.28.1.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://www.openssl.org/source/openssl-1.0.2t.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://github.com/nghttp2/nghttp2/releases/download/v1.36.0/nghttp2-1.36.0.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "http://web.mit.edu/kerberos/dist/krb5/1.15/krb5-1.15.2.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://www.libssh2.org/download/libssh2-1.9.0.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/bison/bison-3.3.2.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/gmp/gmp-6.1.2.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "http://mirror.easyname.at/nongnu/lzip/lzip-1.21.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/gettext/gettext-0.19.8.1.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/mpfr/mpfr-4.0.2.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/mpc/mpc-1.1.0.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://fossies.org/linux/misc/old/libelf-0.8.13.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "http://isl.gforge.inria.fr/isl-0.17.1.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/texinfo/texinfo-6.5.tar.xz"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/which/which-2.21.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "http://dist.schmorp.de/libev/Attic/libev-4.25.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://c-ares.haxx.se/download/c-ares-1.15.0.tar.gz"
+ },
+ {
+ "type": "url",
+ "url": "https://salsa.debian.org/debian/keyutils/raw/4cecffcb8e2a2aa4ef41777ed40e4e4bcfb2e5bf/debian/patches/Make-build-reproducible.patch"
+ },
+ {
+ "type": "url",
+ "url": "https://people.redhat.com/dhowells/keyutils/keyutils-1.6.tar.bz2"
+ },
+ {
+ "type": "url",
+ "url": "https://ftpmirror.gnu.org/m4/m4-1.4.18.tar.bz2"
+ }
+ ],
+ "version": 1
+}
diff --git a/swh/lister/functional_package/tests/test_lister.py b/swh/lister/functional_package/tests/test_lister.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/tests/test_lister.py
@@ -0,0 +1,26 @@
+# Copyright (C) 2019 the Software Heritage developers
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def test_lister_no_page_check_results(lister_functional_package,
+ requests_mock_datadir):
+ lister = lister_functional_package
+# lister.url = "http://sources.nixos.org/packages.json"
+ lister.run()
+
+ r = lister.scheduler.search_tasks(task_type='load-tar')
+ assert len(r) == 76
+
+ for row in r:
+ print(row)
+ assert row['type'] == 'load-tar'
+ # arguments check
+ args = row['arguments']['args']
+ assert len(args) == 1
+
+ # kwargs
+ kwargs = row['arguments']['kwargs']
+ assert list(kwargs.keys()) == ['tarballs']
+
+ assert row['policy'] == 'oneshot'
diff --git a/swh/lister/functional_package/tests/test_tasks.py b/swh/lister/functional_package/tests/test_tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/lister/functional_package/tests/test_tasks.py
@@ -0,0 +1,27 @@
+from unittest.mock import patch
+
+
+def test_ping(swh_app, celery_session_worker):
+ res = swh_app.send_task(
+ 'swh.lister.functional_package.tasks.ping')
+ assert res
+ res.wait()
+ assert res.successful()
+ assert res.result == 'OK'
+
+
+@patch('swh.lister.functional_package.tasks.FunctionalPackageLister')
+def test_lister(lister, swh_app, celery_session_worker):
+ # setup the mocked JSONLister
+ lister.return_value = lister
+ lister.run.return_value = None
+
+ res = swh_app.send_task(
+ 'swh.lister.functional_package.tasks.FunctionalPackageListerTask')
+ assert res
+ res.wait()
+ assert res.successful()
+
+ lister.assert_called_once_with()
+ lister.db_last_index.assert_not_called()
+ lister.run.assert_called_once_with()
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Sun, Aug 17, 7:57 PM (6 d, 6 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3216384
Attached To
D2025: [wip] swh.lister.functionalPackages: add lister getting sources from a JSON file
Event Timeline
Log In to Comment