Page MenuHomeSoftware Heritage
Files F7123007

D7699.diff
No OneTemporary
Actions

D7699.diff
View Options

diff --git a/swh/loader/mercurial/loader.py b/swh/loader/mercurial/loader.py
--- a/swh/loader/mercurial/loader.py
+++ b/swh/loader/mercurial/loader.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2020-2021 The Software Heritage developers
+# Copyright (C) 2020-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -793,7 +793,7 @@
visit_date: Optional[datetime] = None,
archive_path: str = None,
temp_directory: str = "/tmp",
- max_content_size: Optional[int] = None,
+ **kwargs: Any,
):
super().__init__(
storage=storage,
@@ -801,7 +801,7 @@
visit_date=visit_date,
logging_class="swh.loader.mercurial.loader.ArchiveLoader",
temp_directory=temp_directory,
- max_content_size=max_content_size,
+ **kwargs,
)
self.archive_extract_temp_dir = None
self.archive_path = archive_path
diff --git a/swh/loader/mercurial/tasks.py b/swh/loader/mercurial/tasks.py
--- a/swh/loader/mercurial/tasks.py
+++ b/swh/loader/mercurial/tasks.py
@@ -3,8 +3,6 @@
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from typing import Optional
-
from celery import shared_task
from swh.loader.core.utils import parse_visit_date
@@ -12,10 +10,14 @@
from .loader import HgArchiveLoader, HgLoader
+def _process_kwargs(kwargs):
+ if "visit_date" in kwargs:
+ kwargs["visit_date"] = parse_visit_date(kwargs["visit_date"])
+ return kwargs
+
+
@shared_task(name=__name__ + ".LoadMercurial")
-def load_hg(
- *, url: str, directory: Optional[str] = None, visit_date: Optional[str] = None
-):
+def load_hg(**kwargs):
"""Mercurial repository loading
Import a mercurial tarball into swh.
@@ -23,21 +25,15 @@
Args: see :func:`HgLoader` constructor.
"""
- loader = HgLoader.from_configfile(
- url=url, directory=directory, visit_date=parse_visit_date(visit_date)
- )
+ loader = HgLoader.from_configfile(**_process_kwargs(kwargs))
return loader.load()
@shared_task(name=__name__ + ".LoadArchiveMercurial")
-def load_hg_from_archive(
- *, url: str, archive_path: Optional[str] = None, visit_date: Optional[str] = None
-):
+def load_hg_from_archive(**kwargs):
"""Import a mercurial tarball into swh.
Args: see :func:`HgArchiveLoader` constructor.
"""
- loader = HgArchiveLoader.from_configfile(
- url=url, archive_path=archive_path, visit_date=parse_visit_date(visit_date)
- )
+ loader = HgArchiveLoader.from_configfile(**_process_kwargs(kwargs))
return loader.load()
diff --git a/swh/loader/mercurial/tests/test_tasks.py b/swh/loader/mercurial/tests/test_tasks.py
--- a/swh/loader/mercurial/tests/test_tasks.py
+++ b/swh/loader/mercurial/tests/test_tasks.py
@@ -1,22 +1,67 @@
-# Copyright (C) 2018-2021 The Software Heritage developers
+# Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import uuid
+
+import pytest
+
+from swh.scheduler.model import ListedOrigin, Lister
+from swh.scheduler.utils import create_origin_task_dict
+
+
+@pytest.fixture(autouse=True)
+def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config):
+ pass
+
+
+@pytest.fixture
+def hg_lister():
+ return Lister(name="hg-lister", instance_name="example", id=uuid.uuid4())
+
+
+@pytest.fixture
+def hg_listed_origin(hg_lister):
+ return ListedOrigin(
+ lister_id=hg_lister.id, url="https://hg.example.org/repo", visit_type="hg"
+ )
+
def test_loader(
- mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker
+ mocker,
+ swh_scheduler_celery_app,
):
mock_loader = mocker.patch("swh.loader.mercurial.loader.HgLoader.load")
mock_loader.return_value = {"status": "eventful"}
res = swh_scheduler_celery_app.send_task(
"swh.loader.mercurial.tasks.LoadMercurial",
- kwargs={
- "url": "origin_url",
- "directory": "/some/repo",
- "visit_date": "now",
- },
+ kwargs={"url": "origin_url", "visit_date": "now"},
+ )
+
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {"status": "eventful"}
+ mock_loader.assert_called_once_with()
+
+
+def test_loader_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ hg_lister,
+ hg_listed_origin,
+):
+ mock_loader = mocker.patch("swh.loader.mercurial.loader.HgLoader.load")
+ mock_loader.return_value = {"status": "eventful"}
+
+ task_dict = create_origin_task_dict(hg_listed_origin, hg_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.mercurial.tasks.LoadMercurial",
+ kwargs=task_dict["arguments"]["kwargs"],
)
assert res
@@ -28,7 +73,8 @@
def test_archive_loader(
- mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker
+ mocker,
+ swh_scheduler_celery_app,
):
mock_loader = mocker.patch("swh.loader.mercurial.loader.HgArchiveLoader.load")
mock_loader.return_value = {"status": "uneventful"}
@@ -47,3 +93,30 @@
assert res.result == {"status": "uneventful"}
mock_loader.assert_called_once_with()
+
+
+def test_archive_loader_for_listed_origin(
+ mocker,
+ swh_scheduler_celery_app,
+ hg_lister,
+ hg_listed_origin,
+):
+ mock_loader = mocker.patch("swh.loader.mercurial.loader.HgArchiveLoader.load")
+ mock_loader.return_value = {"status": "uneventful"}
+
+ hg_listed_origin.extra_loader_arguments = {
+ "archive_path": "/some/tar.tgz",
+ }
+
+ task_dict = create_origin_task_dict(hg_listed_origin, hg_lister)
+
+ res = swh_scheduler_celery_app.send_task(
+ "swh.loader.mercurial.tasks.LoadArchiveMercurial",
+ kwargs=task_dict["arguments"]["kwargs"],
+ )
+ assert res
+ res.wait()
+ assert res.successful()
+
+ assert res.result == {"status": "uneventful"}
+ mock_loader.assert_called_once_with()

File Metadata

Mime Type
text/plain
Expires
Tue, Dec 17, 3:38 PM (2 d, 21 h ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3229694

Event Timeline

About · Developer information · Legal