diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,3 +5,4 @@ recursive-include swh/loader/package/tests/ *.tar.gz recursive-include swh py.typed recursive-include swh/loader/package/tests/data/ * +recursive-include swh/loader/package/*/tests/data/ * diff --git a/swh/loader/package/tests/conftest.py b/conftest.py rename from swh/loader/package/tests/conftest.py rename to conftest.py --- a/swh/loader/package/tests/conftest.py +++ b/conftest.py @@ -59,4 +59,5 @@ def celery_includes(): return [ 'swh.loader.package.tasks', + 'swh.loader.package.archive.tasks', ] diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ include_package_data=True, entry_points=''' [swh.workers] - loader.package=swh.loader.package:register + loader.archive=swh.loader.package.archive:register ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/loader/package/archive/__init__.py b/swh/loader/package/archive/__init__.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/archive/__init__.py @@ -0,0 +1,14 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register() -> Mapping[str, Any]: + """Register the current worker module's definition""" + return { + 'task_modules': [f'{__name__}.tasks'], + } diff --git a/swh/loader/package/archive.py b/swh/loader/package/archive/loader.py rename from swh/loader/package/archive.py rename to swh/loader/package/archive/loader.py diff --git a/swh/loader/package/archive/tasks.py b/swh/loader/package/archive/tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/archive/tasks.py @@ -0,0 +1,15 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.package.archive.loader import ArchiveLoader + + +@shared_task(name=__name__ + '.LoadArchive') +def load_archive_files(url=None, artifacts=None, identity_artifact_keys=None): + """Load archive's artifacts (e.g gnu, etc...)""" + return ArchiveLoader(url, artifacts, + identity_artifact_keys=identity_artifact_keys).load() diff --git a/swh/loader/package/archive/tests/__init__.py b/swh/loader/package/archive/tests/__init__.py new file mode 100644 diff --git a/swh/loader/package/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.1.0.tar.gz b/swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.1.0.tar.gz rename from swh/loader/package/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.1.0.tar.gz rename to swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.1.0.tar.gz diff --git a/swh/loader/package/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.2.0.tar.gz b/swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.2.0.tar.gz rename from swh/loader/package/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.2.0.tar.gz rename to swh/loader/package/archive/tests/data/https_ftp.gnu.org/gnu_8sync_8sync-0.2.0.tar.gz diff --git a/swh/loader/package/tests/test_archive.py b/swh/loader/package/archive/tests/test_archive.py rename from swh/loader/package/tests/test_archive.py rename to swh/loader/package/archive/tests/test_archive.py --- a/swh/loader/package/tests/test_archive.py +++ b/swh/loader/package/archive/tests/test_archive.py @@ -5,7 +5,7 @@ from swh.model.hashutil import hash_to_bytes -from swh.loader.package.archive import ArchiveLoader, artifact_identity +from swh.loader.package.archive.loader import ArchiveLoader, artifact_identity from swh.loader.package.tests.common import ( check_snapshot, check_metadata_paths, get_stats ) diff --git a/swh/loader/package/archive/tests/test_tasks.py b/swh/loader/package/archive/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/package/archive/tests/test_tasks.py @@ -0,0 +1,21 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from unittest.mock import patch + + +@patch('swh.loader.package.archive.loader.ArchiveLoader.load') +def test_archive_loader( + mock_loader, swh_app, celery_session_worker, swh_config): + mock_loader.return_value = {'status': 'eventful'} + + res = swh_app.send_task( + 'swh.loader.package.archive.tasks.LoadArchive', + (), dict(url='some-url', artifacts=[])) + assert res + res.wait() + assert res.successful() + + assert res.result == {'status': 'eventful'} diff --git a/swh/loader/package/tasks.py b/swh/loader/package/tasks.py --- a/swh/loader/package/tasks.py +++ b/swh/loader/package/tasks.py @@ -9,14 +9,6 @@ from swh.loader.package.deposit import DepositLoader from swh.loader.package.npm import NpmLoader from swh.loader.package.pypi import PyPILoader -from swh.loader.package.archive import ArchiveLoader - - -@shared_task(name=__name__ + '.LoadArchive') -def load_archive_files(url=None, artifacts=None, identity_artifact_keys=None): - """Load archive's artifacts (e.g gnu, etc...)""" - return ArchiveLoader(url, artifacts, - identity_artifact_keys=identity_artifact_keys).load() @shared_task(name=__name__ + '.LoadDebian') diff --git a/swh/loader/package/tests/test_tasks.py b/swh/loader/package/tests/test_tasks.py --- a/swh/loader/package/tests/test_tasks.py +++ b/swh/loader/package/tests/test_tasks.py @@ -6,21 +6,6 @@ from unittest.mock import patch -@patch('swh.loader.package.archive.ArchiveLoader.load') -def test_gnu_loader( - mock_loader, swh_app, celery_session_worker, swh_config): - mock_loader.return_value = {'status': 'eventful'} - - res = swh_app.send_task( - 'swh.loader.package.tasks.LoadArchive', - (), dict(url='some-url', artifacts=[])) - assert res - res.wait() - assert res.successful() - - assert res.result == {'status': 'eventful'} - - @patch('swh.loader.package.debian.DebianLoader.load') def test_debian_loader( mock_loader, swh_app, celery_session_worker, swh_config):