diff --git a/swh/lister/sourceforge/tasks.py b/swh/lister/sourceforge/tasks.py index 6ab0815..fb111be 100644 --- a/swh/lister/sourceforge/tasks.py +++ b/swh/lister/sourceforge/tasks.py @@ -1,20 +1,26 @@ # Copyright (C) 2019-2021 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Dict from celery import shared_task from swh.lister.sourceforge.lister import SourceForgeLister @shared_task(name=__name__ + ".FullSourceForgeLister") def list_sourceforge_full() -> Dict[str, int]: """Full update of a SourceForge instance""" return SourceForgeLister.from_configfile().run().dict() +@shared_task(name=__name__ + ".IncrementalSourceForgeLister") +def list_sourceforge_incremental() -> Dict[str, int]: + """Full update of a SourceForge instance""" + return SourceForgeLister.from_configfile(incremental=True).run().dict() + + @shared_task(name=__name__ + ".ping") def _ping(): return "OK" diff --git a/swh/lister/sourceforge/tests/test_tasks.py b/swh/lister/sourceforge/tests/test_tasks.py index 3baad10..51752a9 100644 --- a/swh/lister/sourceforge/tests/test_tasks.py +++ b/swh/lister/sourceforge/tests/test_tasks.py @@ -1,34 +1,56 @@ # Copyright (C) 2019-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.lister.pattern import ListerStats +lister_module = "swh.lister.sourceforge.tasks.SourceForgeLister" + def test_sourceforge_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): res = swh_scheduler_celery_app.send_task("swh.lister.sourceforge.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" def test_sourceforge_full_lister_task( swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker ): stats = ListerStats(pages=10, origins=900) - mock_lister = mocker.patch("swh.lister.sourceforge.tasks.SourceForgeLister") + mock_lister = mocker.patch(lister_module) mock_lister.from_configfile.return_value = mock_lister mock_lister.run.return_value = stats res = swh_scheduler_celery_app.send_task( "swh.lister.sourceforge.tasks.FullSourceForgeLister" ) assert res res.wait() assert res.successful() mock_lister.from_configfile.assert_called_once() mock_lister.run.assert_called_once() assert res.result == stats.dict() + + +def test_incremental_listing( + swh_scheduler_celery_app, swh_scheduler_celery_worker, mocker +): + stats = ListerStats(pages=1, origins=90) + mock_lister = mocker.patch(lister_module) + mock_lister.from_configfile.return_value = mock_lister + mock_lister.run.return_value = stats + + res = swh_scheduler_celery_app.send_task( + "swh.lister.sourceforge.tasks.IncrementalSourceForgeLister" + ) + assert res + res.wait() + assert res.successful() + + mock_lister.from_configfile.assert_called_once() + mock_lister.run.assert_called_once() + assert res.result == stats.dict()