diff --git a/swh/loader/svn/tasks.py b/swh/loader/svn/tasks.py index a9104f5..4f60bfb 100644 --- a/swh/loader/svn/tasks.py +++ b/swh/loader/svn/tasks.py @@ -1,38 +1,46 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from celery import shared_task +from swh.loader.core.utils import parse_visit_date + from .loader import SvnLoader, SvnLoaderFromDumpArchive, SvnLoaderFromRemoteDump +def _process_kwargs(kwargs): + if "visit_date" in kwargs: + kwargs["visit_date"] = parse_visit_date(kwargs["visit_date"]) + return kwargs + + @shared_task(name=__name__ + ".LoadSvnRepository") -def load_svn(*args, **kwargs): +def load_svn(**kwargs): """Import a svn repository""" - loader = SvnLoader.from_configfile(*args, **kwargs) + loader = SvnLoader.from_configfile(**_process_kwargs(kwargs)) return loader.load() @shared_task(name=__name__ + ".MountAndLoadSvnRepository") -def load_svn_from_archive(*args, **kwargs): +def load_svn_from_archive(**kwargs): """ 1. Mount an svn dump from archive as a local svn repository 2. Load it through the svn loader 3. Clean up mounted svn repository archive """ - loader = SvnLoaderFromDumpArchive.from_configfile(*args, **kwargs) + loader = SvnLoaderFromDumpArchive.from_configfile(**_process_kwargs(kwargs)) return loader.load() @shared_task(name=__name__ + ".DumpMountAndLoadSvnRepository") -def load_svn_from_remote_dump(*args, **kwargs): +def load_svn_from_remote_dump(**kwargs): """ 1. Mount a remote svn dump as a local svn repository. 2. Load it through the svn loader. 3. Clean up mounted svn repository archive. """ - loader = SvnLoaderFromRemoteDump.from_configfile(*args, **kwargs) + loader = SvnLoaderFromRemoteDump.from_configfile(**_process_kwargs(kwargs)) return loader.load() diff --git a/swh/loader/svn/tests/test_task.py b/swh/loader/svn/tests/test_task.py index a4a3896..efd88e0 100644 --- a/swh/loader/svn/tests/test_task.py +++ b/swh/loader/svn/tests/test_task.py @@ -1,152 +1,156 @@ # Copyright (C) 2019-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import uuid import pytest from swh.scheduler.model import ListedOrigin, Lister from swh.scheduler.utils import create_origin_task_dict @pytest.fixture(autouse=True) def celery_worker_and_swh_config(swh_scheduler_celery_worker, swh_config): pass @pytest.fixture def svn_lister(): return Lister(name="svn-lister", instance_name="example", id=uuid.uuid4()) @pytest.fixture def svn_listed_origin(svn_lister): return ListedOrigin( lister_id=svn_lister.id, url="svn://example.org/repo", visit_type="svn" ) @pytest.fixture def task_dict(svn_lister, svn_listed_origin): return create_origin_task_dict(svn_listed_origin, svn_lister) def test_svn_loader( mocker, swh_scheduler_celery_app, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoader.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.LoadSvnRepository", - kwargs=dict(url="some-technical-url", origin_url="origin-url"), + kwargs=dict( + url="some-technical-url", origin_url="origin-url", visit_date="now" + ), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_for_listed_origin( mocker, swh_scheduler_celery_app, task_dict, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoader.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.LoadSvnRepository", args=task_dict["arguments"]["args"], kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_dump( mocker, swh_scheduler_celery_app, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromDumpArchive.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.MountAndLoadSvnRepository", - kwargs=dict(url="some-url", archive_path="some-path"), + kwargs=dict(url="some-url", archive_path="some-path", visit_date="now"), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_dump_for_listed_origin( mocker, swh_scheduler_celery_app, svn_lister, svn_listed_origin, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromDumpArchive.load") mock_loader.return_value = {"status": "eventful"} svn_listed_origin.extra_loader_arguments = {"archive_path": "some-path"} task_dict = create_origin_task_dict(svn_listed_origin, svn_lister) res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.MountAndLoadSvnRepository", args=task_dict["arguments"]["args"], kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_remote_dump( mocker, swh_scheduler_celery_app, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromRemoteDump.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.DumpMountAndLoadSvnRepository", - kwargs=dict(url="some-remote-dump-url", origin_url="origin-url"), + kwargs=dict( + url="some-remote-dump-url", origin_url="origin-url", visit_date="now" + ), ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"} def test_svn_loader_from_remote_dump_for_listed_origin( mocker, swh_scheduler_celery_app, task_dict, ): mock_loader = mocker.patch("swh.loader.svn.loader.SvnLoaderFromRemoteDump.load") mock_loader.return_value = {"status": "eventful"} res = swh_scheduler_celery_app.send_task( "swh.loader.svn.tasks.DumpMountAndLoadSvnRepository", args=task_dict["arguments"]["args"], kwargs=task_dict["arguments"]["kwargs"], ) assert res res.wait() assert res.successful() assert res.result == {"status": "eventful"}