diff --git a/swh/loader/mercurial/loader.py b/swh/loader/mercurial/loader.py --- a/swh/loader/mercurial/loader.py +++ b/swh/loader/mercurial/loader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2020-2021 The Software Heritage developers +# Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -793,7 +793,7 @@ visit_date: Optional[datetime] = None, archive_path: str = None, temp_directory: str = "/tmp", - max_content_size: Optional[int] = None, + **kwargs: Any, ): super().__init__( storage=storage, @@ -801,7 +801,7 @@ visit_date=visit_date, logging_class="swh.loader.mercurial.loader.ArchiveLoader", temp_directory=temp_directory, - max_content_size=max_content_size, + **kwargs, ) self.archive_extract_temp_dir = None self.archive_path = archive_path diff --git a/swh/loader/mercurial/tasks.py b/swh/loader/mercurial/tasks.py --- a/swh/loader/mercurial/tasks.py +++ b/swh/loader/mercurial/tasks.py @@ -3,8 +3,6 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Optional - from celery import shared_task from swh.loader.core.utils import parse_visit_date @@ -12,10 +10,14 @@ from .loader import HgArchiveLoader, HgLoader +def _process_kwargs(kwargs): + if "visit_date" in kwargs: + kwargs["visit_date"] = parse_visit_date(kwargs["visit_date"]) + return kwargs + + @shared_task(name=__name__ + ".LoadMercurial") -def load_hg( - *, url: str, directory: Optional[str] = None, visit_date: Optional[str] = None -): +def load_hg(**kwargs): """Mercurial repository loading Import a mercurial tarball into swh. @@ -23,21 +25,15 @@ Args: see :func:`HgLoader` constructor. """ - loader = HgLoader.from_configfile( - url=url, directory=directory, visit_date=parse_visit_date(visit_date) - ) + loader = HgLoader.from_configfile(**_process_kwargs(kwargs)) return loader.load() @shared_task(name=__name__ + ".LoadArchiveMercurial") -def load_hg_from_archive( - *, url: str, archive_path: Optional[str] = None, visit_date: Optional[str] = None -): +def load_hg_from_archive(**kwargs): """Import a mercurial tarball into swh. Args: see :func:`HgArchiveLoader` constructor. """ - loader = HgArchiveLoader.from_configfile( - url=url, archive_path=archive_path, visit_date=parse_visit_date(visit_date) - ) + loader = HgArchiveLoader.from_configfile(**_process_kwargs(kwargs)) return loader.load() diff --git a/swh/loader/mercurial/tests/test_tasks.py b/swh/loader/mercurial/tests/test_tasks.py --- a/swh/loader/mercurial/tests/test_tasks.py +++ b/swh/loader/mercurial/tests/test_tasks.py @@ -1,22 +1,67 @@ -# Copyright (C) 2018-2021 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import uuid + +import pytest + +from swh.scheduler.model import ListedOrigin, Lister +from swh.scheduler.utils import create_origin_task_dict + + +@pytest.fixture(autouse=True) +def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): + pass + + +@pytest.fixture +def hg_lister(): + return Lister(name="hg-lister", instance_name="example", id=uuid.uuid4()) + + +@pytest.fixture +def hg_listed_origin(hg_lister): + return ListedOrigin( + lister_id=hg_lister.id, url="https://hg.example.org/repo", visit_type="hg" + ) + def test_loader( - mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch("swh.loader.mercurial.loader.HgLoader.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.mercurial.tasks.LoadMercurial", - kwargs={ - "url": "origin_url", - "directory": "/some/repo", - "visit_date": "now", - }, + kwargs={"url": "origin_url", "visit_date": "now"}, + ) + + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "eventful"} + mock_loader.assert_called_once_with() + + +def test_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + hg_lister, + hg_listed_origin, +): + mock_loader = mocker.patch("swh.loader.mercurial.loader.HgLoader.load") + mock_loader.return_value = {"status": "eventful"} + + task_dict = create_origin_task_dict(hg_listed_origin, hg_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.mercurial.tasks.LoadMercurial", + kwargs=task_dict["arguments"]["kwargs"], ) assert res @@ -28,7 +73,8 @@ def test_archive_loader( - mocker, swh_config, swh_scheduler_celery_app, swh_scheduler_celery_worker + mocker, + swh_scheduler_celery_app, ): mock_loader = mocker.patch("swh.loader.mercurial.loader.HgArchiveLoader.load") mock_loader.return_value = {"status": "uneventful"} @@ -47,3 +93,30 @@ assert res.result == {"status": "uneventful"} mock_loader.assert_called_once_with() + + +def test_archive_loader_for_listed_origin( + mocker, + swh_scheduler_celery_app, + hg_lister, + hg_listed_origin, +): + mock_loader = mocker.patch("swh.loader.mercurial.loader.HgArchiveLoader.load") + mock_loader.return_value = {"status": "uneventful"} + + hg_listed_origin.extra_loader_arguments = { + "archive_path": "/some/tar.tgz", + } + + task_dict = create_origin_task_dict(hg_listed_origin, hg_lister) + + res = swh_scheduler_celery_app.send_task( + "swh.loader.mercurial.tasks.LoadArchiveMercurial", + kwargs=task_dict["arguments"]["kwargs"], + ) + assert res + res.wait() + assert res.successful() + + assert res.result == {"status": "uneventful"} + mock_loader.assert_called_once_with()