diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,3 +1,4 @@ pytest requests_mock swh-core[testing] +swh-scheduler[testing] diff --git a/swh/loader/package/tasks.py b/swh/loader/package/tasks.py --- a/swh/loader/package/tasks.py +++ b/swh/loader/package/tasks.py @@ -3,10 +3,35 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from celery import current_app as app +from celery import shared_task + +from swh.loader.package.debian import DebianLoader +from swh.loader.package.deposit import DepositLoader from swh.loader.package.gnu import GNULoader +from swh.loader.package.npm import NpmLoader +from swh.loader.package.pypi import PyPILoader + + +@shared_task(name=__name__ + '.LoadDebian') +def load_debian(*, url, date, packages): + return DebianLoader(url, date, packages).load() + + +@shared_task(name=__name__ + '.LoadDeposit') +def load_deposit(*, url, deposit_id): + return DepositLoader(url, deposit_id).load() + + +@shared_task(name=__name__ + '.LoadGNU') +def load_gnu(*, url, tarballs): + return GNULoader(url, tarballs).load() + + +@shared_task(name=__name__ + '.LoadNpm') +def load_npm(*, package_name, package_url, package_metadata_url): + return NpmLoader(package_name, package_url, package_metadata_url).load() -@app.task(name=__name__ + '.LoadGNU') -def load_gnu(name, origin_url=None, tarballs=None): - return GNULoader(origin_url, tarballs).load() +@shared_task(name=__name__ + '.LoadPyPI') +def load_pypi(*, url=None): + return PyPILoader(url).load() diff --git a/swh/loader/package/tests/conftest.py b/swh/loader/package/tests/conftest.py --- a/swh/loader/package/tests/conftest.py +++ b/swh/loader/package/tests/conftest.py @@ -8,6 +8,7 @@ import yaml from swh.storage.tests.conftest import * # noqa +from swh.scheduler.tests.conftest import * # noqa @pytest.fixture @@ -41,3 +42,10 @@ """ os.environ['http_proxy'] = 'http://localhost:999' os.environ['https_proxy'] = 'http://localhost:999' + + +@pytest.fixture(scope='session') # type: ignore # expected redefinition +def celery_includes(): + return [ + 'swh.loader.package.tasks', + ] diff --git a/swh/loader/package/tests/test_tasks.py b/swh/loader/package/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/tests/test_tasks.py @@ -0,0 +1,83 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from unittest.mock import patch + + +@patch('swh.loader.package.debian.DebianLoader.load') +def test_debian_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadDebian', + (), dict(url='some-url', date='some-date', packages={})) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.deposit.DepositLoader.load') +def test_deposit_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadDeposit', + (), dict(url='some-url', deposit_id='some-d-id')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.gnu.GNULoader.load') +def test_gnu_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadGNU', + (), dict(url='some-url', tarballs=[])) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.npm.NpmLoader.load') +def test_npm_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadNpm', + (), dict(package_name='some-package', + package_url='some', + package_metadata_url='something')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} + + +@patch('swh.loader.package.pypi.PyPILoader.load') +def test_pypi_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.tasks.LoadPyPI', + (), dict(url='some-url')) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'}