diff --git a/docs/README.rst b/docs/README.rst --- a/docs/README.rst +++ b/docs/README.rst @@ -5,9 +5,12 @@ helpers used by :term:`loaders `. The main entry points are classes: -- :class:`swh.loader.core.loader.BaseLoader` for loaders (e.g. svn) -- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. hg, git, ...) +- :class:`swh.loader.core.loader.BaseLoader` for VCS loaders (e.g. svn) +- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. git, ...) +- :class:`swh.loader.core.loader.ContentLoader` for Content loader +- :class:`swh.loader.core.loader.DirectoryLoader` for Directory loader - :class:`swh.loader.package.loader.PackageLoader` for Package loaders (e.g. PyPI, Npm, ...) +- ... Package loaders --------------- diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -55,6 +55,8 @@ [swh.cli.subcommands] loader=swh.loader.cli [swh.workers] + loader.content=swh.loader.core:register_content + loader.directory=swh.loader.core:register_directory loader.arch=swh.loader.package.arch:register loader.archive=swh.loader.package.archive:register loader.aur=swh.loader.package.aur:register diff --git a/swh/loader/core/__init__.py b/swh/loader/core/__init__.py --- a/swh/loader/core/__init__.py +++ b/swh/loader/core/__init__.py @@ -0,0 +1,27 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from typing import Any, Mapping + + +def register_content() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from swh.loader.core.loader import ContentLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": ContentLoader, + } + + +def register_directory() -> Mapping[str, Any]: + """Register the current worker module's definition""" + from swh.loader.core.loader import DirectoryLoader + + return { + "task_modules": [f"{__name__}.tasks"], + "loader": DirectoryLoader, + } diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py --- a/swh/loader/core/loader.py +++ b/swh/loader/core/loader.py @@ -667,12 +667,13 @@ def __init__( self, - *args, + storage: StorageInterface, + url: str, checksums: Dict[str, str], fallback_urls: List[str] = None, **kwargs, ): - super().__init__(*args, **kwargs) + super().__init__(storage, url, **kwargs) self.snapshot: Optional[Snapshot] = None self.checksums = checksums fallback_urls_ = fallback_urls or [] diff --git a/swh/loader/core/tasks.py b/swh/loader/core/tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/core/tasks.py @@ -0,0 +1,20 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from celery import shared_task + +from swh.loader.core.loader import ContentLoader, DirectoryLoader + + +@shared_task(name=__name__ + ".LoadContent") +def load_content(**kwargs): + """Load Content package""" + return ContentLoader.from_configfile(**kwargs).load() + + +@shared_task(name=__name__ + ".LoadDirectory") +def load_directory(**kwargs): + """Load Content package""" + return DirectoryLoader.from_configfile(**kwargs).load() diff --git a/swh/loader/core/tests/test_tasks.py b/swh/loader/core/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/loader/core/tests/test_tasks.py @@ -0,0 +1,53 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + +NAMESPACE = "swh.loader.core" + + +@pytest.fixture +def nixguix_lister(): + return Lister(name="nixguix", instance_name="example", id=uuid.uuid4()) + + +@pytest.mark.parametrize("loader_name", ["Content", "Directory"]) +def test_tasks_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + swh_scheduler_celery_worker, + swh_config, + nixguix_lister, + loader_name, +): + mock_load = mocker.patch(f"{NAMESPACE}.loader.{loader_name}Loader.load") + mock_load.return_value = {"status": "eventful"} + + listed_origin = ListedOrigin( + lister_id=nixguix_lister.id, + url="https://example.org/artifact/artifact", + visit_type=loader_name.lower(), + extra_loader_arguments={ + "fallback_urls": ["https://example.org/mirror/artifact-0.0.1.pkg.xz"], + "checksums": {"sha256": "some-valid-checksum"}, + }, + ) + + task_dict = create_origin_task_dict(listed_origin, nixguix_lister) + + res = swh_scheduler_celery_app.send_task( + f"{NAMESPACE}.tasks.Load{loader_name}", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + assert mock_load.called + assert res.result == {"status": "eventful"}