diff --git a/requirements-test.txt b/requirements-test.txt index 5847ac3..a582a1e 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ pytest requests_mock swh-core[testing] +swh-scheduler[testing] diff --git a/swh/loader/package/tasks.py b/swh/loader/package/tasks.py index 9062231..c39ec40 100644 --- a/swh/loader/package/tasks.py +++ b/swh/loader/package/tasks.py @@ -1,12 +1,37 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from celery import current_app as app +from celery import shared_task + +from swh.loader.package.debian import DebianLoader +from swh.loader.package.deposit import DepositLoader from swh.loader.package.gnu import GNULoader +from swh.loader.package.npm import NpmLoader +from swh.loader.package.pypi import PyPILoader + + +@shared_task(name=__name__ + '.LoadDebian') +def load_debian(*, url, date, packages): + return DebianLoader(url, date, packages).load() + + +@shared_task(name=__name__ + '.LoadDeposit') +def load_deposit(*, url, deposit_id): + return DepositLoader(url, deposit_id).load() + + +@shared_task(name=__name__ + '.LoadGNU') +def load_gnu(*, url, tarballs): + return GNULoader(url, tarballs).load() + + +@shared_task(name=__name__ + '.LoadNpm') +def load_npm(*, package_name, package_url, package_metadata_url): + return NpmLoader(package_name, package_url, package_metadata_url).load() -@app.task(name=__name__ + '.LoadGNU') -def load_gnu(name, origin_url=None, tarballs=None): - return GNULoader(origin_url, tarballs).load() +@shared_task(name=__name__ + '.LoadPyPI') +def load_pypi(*, url=None): + return PyPILoader(url).load() diff --git a/swh/loader/package/tests/conftest.py b/swh/loader/package/tests/conftest.py index 8360b06..f80142f 100644 --- a/swh/loader/package/tests/conftest.py +++ b/swh/loader/package/tests/conftest.py @@ -1,43 +1,51 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import pytest import yaml from swh.storage.tests.conftest import * # noqa +from swh.scheduler.tests.conftest import * # noqa @pytest.fixture def swh_config(monkeypatch, swh_storage_postgresql, tmp_path): storage_config = { 'storage': { 'cls': 'local', 'args': { 'db': swh_storage_postgresql.dsn, 'objstorage': { 'cls': 'memory', 'args': {} }, }, }, 'url': 'https://deposit.softwareheritage.org/1/private', } conffile = os.path.join(str(tmp_path), 'loader.yml') with open(conffile, 'w') as f: f.write(yaml.dump(storage_config)) monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile) return conffile @pytest.fixture(autouse=True, scope='session') def swh_proxy(): """Automatically inject this fixture in all tests to ensure no outside connection takes place. """ os.environ['http_proxy'] = 'http://localhost:999' os.environ['https_proxy'] = 'http://localhost:999' + + +@pytest.fixture(scope='session') # type: ignore # expected redefinition +def celery_includes(): + return [ + 'swh.loader.package.tasks', + ] diff --git a/swh/loader/package/tests/test_tasks.py b/swh/loader/package/tests/test_tasks.py new file mode 100644 index 0000000..4c0cca3 --- /dev/null +++ b/swh/loader/package/tests/test_tasks.py @@ -0,0 +1,83 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from unittest.mock import patch + + +@patch('swh.loader.package.debian.DebianLoader.load') +def test_debian_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadDebian', + (), dict(url='some-url', date='some-date', packages={})) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.deposit.DepositLoader.load') +def test_deposit_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadDeposit', + (), dict(url='some-url', deposit_id='some-d-id')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.gnu.GNULoader.load') +def test_gnu_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadGNU', + (), dict(url='some-url', tarballs=[])) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.npm.NpmLoader.load') +def test_npm_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadNpm', + (), dict(package_name='some-package', + package_url='some', + package_metadata_url='something')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.pypi.PyPILoader.load') +def test_pypi_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadPyPI', + (), dict(url='some-url')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'}