Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7123222
D8601.id31075.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
5 KB
Subscribers
None
D8601.id31075.diff
View Options
diff --git a/docs/README.rst b/docs/README.rst
--- a/docs/README.rst
+++ b/docs/README.rst
@@ -5,9 +5,12 @@
helpers used by :term:`loaders <loader>`.
The main entry points are classes:
-- :class:`swh.loader.core.loader.BaseLoader` for loaders (e.g. svn)
-- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. hg, git, ...)
+- :class:`swh.loader.core.loader.BaseLoader` for VCS loaders (e.g. svn)
+- :class:`swh.loader.core.loader.DVCSLoader` for DVCS loaders (e.g. git, ...)
+- :class:`swh.loader.core.loader.ContentLoader` for Content loader
+- :class:`swh.loader.core.loader.DirectoryLoader` for Directory loader
- :class:`swh.loader.package.loader.PackageLoader` for Package loaders (e.g. PyPI, Npm, ...)
+- ...
Package loaders
---------------
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -55,6 +55,8 @@
[swh.cli.subcommands]
loader=swh.loader.cli
[swh.workers]
+ loader.content=swh.loader.core:register_content
+ loader.directory=swh.loader.core:register_directory
loader.arch=swh.loader.package.arch:register
loader.archive=swh.loader.package.archive:register
loader.aur=swh.loader.package.aur:register
diff --git a/swh/loader/core/__init__.py b/swh/loader/core/__init__.py
--- a/swh/loader/core/__init__.py
+++ b/swh/loader/core/__init__.py
@@ -0,0 +1,27 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+from typing import Any, Mapping
+
+
+def register_content() -> Mapping[str, Any]:
+ """Register the current worker module's definition"""
+ from swh.loader.core.loader import ContentLoader
+
+ return {
+ "task_modules": [f"{__name__}.tasks"],
+ "loader": ContentLoader,
+ }
+
+
+def register_directory() -> Mapping[str, Any]:
+ """Register the current worker module's definition"""
+ from swh.loader.core.loader import DirectoryLoader
+
+ return {
+ "task_modules": [f"{__name__}.tasks"],
+ "loader": DirectoryLoader,
+ }
diff --git a/swh/loader/core/loader.py b/swh/loader/core/loader.py
--- a/swh/loader/core/loader.py
+++ b/swh/loader/core/loader.py
@@ -667,12 +667,13 @@
def __init__(
self,
- *args,
+ storage: StorageInterface,
+ url: str,
checksums: Dict[str, str],
fallback_urls: List[str] = None,
**kwargs,
):
- super().__init__(*args, **kwargs)
+ super().__init__(storage, url, **kwargs)
self.snapshot: Optional[Snapshot] = None
self.checksums = checksums
fallback_urls_ = fallback_urls or []
diff --git a/swh/loader/core/tasks.py b/swh/loader/core/tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/core/tasks.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from celery import shared_task
+
+from swh.loader.core.loader import ContentLoader, DirectoryLoader
+
+
+@shared_task(name=__name__ + ".LoadContent")
+def load_content(**kwargs):
+ """Load Content package"""
+ return ContentLoader.from_configfile(**kwargs).load()
+
+
+@shared_task(name=__name__ + ".LoadDirectory")
+def load_directory(**kwargs):
+ """Load Content package"""
+ return DirectoryLoader.from_configfile(**kwargs).load()
diff --git a/swh/loader/core/tests/test_tasks.py b/swh/loader/core/tests/test_tasks.py
new file mode 100644
--- /dev/null
+++ b/swh/loader/core/tests/test_tasks.py
@@ -0,0 +1,53 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import uuid
+
+import pytest
+
+from swh.scheduler.model import ListedOrigin, Lister
+from swh.scheduler.utils import create_origin_task_dict
+
+NAMESPACE = "swh.loader.core"
+
+
+@pytest.fixture
+def nixguix_lister():
+ return Lister(name="nixguix", instance_name="example", id=uuid.uuid4())
+
+
+@pytest.mark.parametrize("loader_name", ["Content", "Directory"])
+def test_tasks_loader_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ swh_scheduler_celery_worker,
+ swh_config,
+ nixguix_lister,
+ loader_name,
+):
+ mock_load = mocker.patch(f"{NAMESPACE}.loader.{loader_name}Loader.load")
+ mock_load.return_value = {"status": "eventful"}
+
+ listed_origin = ListedOrigin(
+ lister_id=nixguix_lister.id,
+ url="https://example.org/artifact/artifact",
+ visit_type=loader_name.lower(),
+ extra_loader_arguments={
+ "fallback_urls": ["https://example.org/mirror/artifact-0.0.1.pkg.xz"],
+ "checksums": {"sha256": "some-valid-checksum"},
+ },
+ )
+
+ task_dict = create_origin_task_dict(listed_origin, nixguix_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ f"{NAMESPACE}.tasks.Load{loader_name}",
+ kwargs=task_dict["arguments"]["kwargs"],
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+ assert mock_load.called
+ assert res.result == {"status": "eventful"}
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Wed, Dec 18, 3:48 AM (1 d, 21 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3218607
Attached To
D8601: {Content|Directory}Loader: Register tasks
Event Timeline
Log In to Comment