Page MenuHomeSoftware Heritage
Files F9749894

No OneTemporary
Actions

View Options

diff --git a/requirements-test.txt b/requirements-test.txt
index 5847ac3..a582a1e 100644
--- a/requirements-test.txt
+++ b/requirements-test.txt
@@ -1,3 +1,4 @@
pytest
requests_mock
swh-core[testing]
+swh-scheduler[testing]
diff --git a/swh/loader/package/tasks.py b/swh/loader/package/tasks.py
index 9062231..c39ec40 100644
--- a/swh/loader/package/tasks.py
+++ b/swh/loader/package/tasks.py
@@ -1,12 +1,37 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from celery import current_app as app
+from celery import shared_task
+
+from swh.loader.package.debian import DebianLoader
+from swh.loader.package.deposit import DepositLoader
from swh.loader.package.gnu import GNULoader
+from swh.loader.package.npm import NpmLoader
+from swh.loader.package.pypi import PyPILoader
+
+
+@shared_task(name=__name__ + '.LoadDebian')
+def load_debian(*, url, date, packages):
+ return DebianLoader(url, date, packages).load()
+
+
+@shared_task(name=__name__ + '.LoadDeposit')
+def load_deposit(*, url, deposit_id):
+ return DepositLoader(url, deposit_id).load()
+
+
+@shared_task(name=__name__ + '.LoadGNU')
+def load_gnu(*, url, tarballs):
+ return GNULoader(url, tarballs).load()
+
+
+@shared_task(name=__name__ + '.LoadNpm')
+def load_npm(*, package_name, package_url, package_metadata_url):
+ return NpmLoader(package_name, package_url, package_metadata_url).load()
-@app.task(name=__name__ + '.LoadGNU')
-def load_gnu(name, origin_url=None, tarballs=None):
- return GNULoader(origin_url, tarballs).load()
+@shared_task(name=__name__ + '.LoadPyPI')
+def load_pypi(*, url=None):
+ return PyPILoader(url).load()
diff --git a/swh/loader/package/tests/conftest.py b/swh/loader/package/tests/conftest.py
index 8360b06..f80142f 100644
--- a/swh/loader/package/tests/conftest.py
+++ b/swh/loader/package/tests/conftest.py
@@ -1,43 +1,51 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
import pytest
import yaml
from swh.storage.tests.conftest import * # noqa
+from swh.scheduler.tests.conftest import * # noqa
@pytest.fixture
def swh_config(monkeypatch, swh_storage_postgresql, tmp_path):
storage_config = {
'storage': {
'cls': 'local',
'args': {
'db': swh_storage_postgresql.dsn,
'objstorage': {
'cls': 'memory',
'args': {}
},
},
},
'url': 'https://deposit.softwareheritage.org/1/private',
}
conffile = os.path.join(str(tmp_path), 'loader.yml')
with open(conffile, 'w') as f:
f.write(yaml.dump(storage_config))
monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile)
return conffile
@pytest.fixture(autouse=True, scope='session')
def swh_proxy():
"""Automatically inject this fixture in all tests to ensure no outside
connection takes place.
"""
os.environ['http_proxy'] = 'http://localhost:999'
os.environ['https_proxy'] = 'http://localhost:999'
+
+
+@pytest.fixture(scope='session') # type: ignore # expected redefinition
+def celery_includes():
+ return [
+ 'swh.loader.package.tasks',
+ ]
diff --git a/swh/loader/package/tests/test_tasks.py b/swh/loader/package/tests/test_tasks.py
new file mode 100644
index 0000000..4c0cca3
--- /dev/null
+++ b/swh/loader/package/tests/test_tasks.py
@@ -0,0 +1,83 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from unittest.mock import patch
+
+
+@patch('swh.loader.package.debian.DebianLoader.load')
+def test_debian_loader(
+ mock_loader, swh_app, celery_session_worker, swh_config):
+ mock_loader.return_value = {'status': 'eventful'}
+
+ res = swh_app.send_task(
+ 'swh.loader.package.tasks.LoadDebian',
+ (), dict(url='some-url', date='some-date', packages={}))
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {'status': 'eventful'}
+
+
+@patch('swh.loader.package.deposit.DepositLoader.load')
+def test_deposit_loader(
+ mock_loader, swh_app, celery_session_worker, swh_config):
+ mock_loader.return_value = {'status': 'eventful'}
+
+ res = swh_app.send_task(
+ 'swh.loader.package.tasks.LoadDeposit',
+ (), dict(url='some-url', deposit_id='some-d-id'))
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {'status': 'eventful'}
+
+
+@patch('swh.loader.package.gnu.GNULoader.load')
+def test_gnu_loader(
+ mock_loader, swh_app, celery_session_worker, swh_config):
+ mock_loader.return_value = {'status': 'eventful'}
+
+ res = swh_app.send_task(
+ 'swh.loader.package.tasks.LoadGNU',
+ (), dict(url='some-url', tarballs=[]))
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {'status': 'eventful'}
+
+
+@patch('swh.loader.package.npm.NpmLoader.load')
+def test_npm_loader(
+ mock_loader, swh_app, celery_session_worker, swh_config):
+ mock_loader.return_value = {'status': 'eventful'}
+
+ res = swh_app.send_task(
+ 'swh.loader.package.tasks.LoadNpm',
+ (), dict(package_name='some-package',
+ package_url='some',
+ package_metadata_url='something'))
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {'status': 'eventful'}
+
+
+@patch('swh.loader.package.pypi.PyPILoader.load')
+def test_pypi_loader(
+ mock_loader, swh_app, celery_session_worker, swh_config):
+ mock_loader.return_value = {'status': 'eventful'}
+
+ res = swh_app.send_task(
+ 'swh.loader.package.tasks.LoadPyPI',
+ (), dict(url='some-url'))
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {'status': 'eventful'}

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 25, 6:24 PM (1 d, 3 h)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3266807

Event Timeline

About · Developer information · Legal