diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..aeb1ec3 --- /dev/null +++ b/conftest.py @@ -0,0 +1,6 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +pytest_plugins = ["swh.scheduler.pytest_plugin"] diff --git a/pytest.ini b/pytest.ini index afa4cf3..7cc9504 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,2 +1,5 @@ [pytest] +# We need this to avoid conflicts between the autoloaded fixture, and the +# manually loaded one. +addopts = -p no:pytest_swh_scheduler norecursedirs = docs diff --git a/requirements-test.txt b/requirements-test.txt index 7cc4593..833da5c 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,5 +1,6 @@ pytest pytest-postgresql requests_mock -testing.postgresql sqlalchemy-stubs +swh.scheduler[testing] >= 0.5 +testing.postgresql diff --git a/swh/lister/bitbucket/tests/test_tasks.py b/swh/lister/bitbucket/tests/test_tasks.py index 9441f09..3f02b9a 100644 --- a/swh/lister/bitbucket/tests/test_tasks.py +++ b/swh/lister/bitbucket/tests/test_tasks.py @@ -1,86 +1,95 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from time import sleep from celery.result import GroupResult from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.bitbucket.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.bitbucket.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.bitbucket.tasks.BitBucketLister") -def test_incremental(lister, swh_app, celery_session_worker): +def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked BitbucketLister lister.return_value = lister lister.db_last_index.return_value = 42 lister.run.return_value = None - res = swh_app.send_task("swh.lister.bitbucket.tasks.IncrementalBitBucketLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.bitbucket.tasks.IncrementalBitBucketLister" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_called_once_with() lister.run.assert_called_once_with(min_bound=42, max_bound=None) @patch("swh.lister.bitbucket.tasks.BitBucketLister") -def test_range(lister, swh_app, celery_session_worker): +def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked BitbucketLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.bitbucket.tasks.RangeBitBucketLister", kwargs=dict(start=12, end=42) ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with(min_bound=12, max_bound=42) @patch("swh.lister.bitbucket.tasks.BitBucketLister") -def test_relister(lister, swh_app, celery_session_worker): +def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked BitbucketLister lister.return_value = lister lister.run.return_value = None lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)] - res = swh_app.send_task("swh.lister.bitbucket.tasks.FullBitBucketRelister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.bitbucket.tasks.FullBitBucketRelister" + ) assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with() # one by the FullBitbucketRelister task # + 5 for the RangeBitbucketLister subtasks assert lister.call_count == 6 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_called_once_with(10000) # lister.run should have been called once per partition interval for i in range(5): assert ( dict(min_bound=10 * i, max_bound=10 * i + 9), ) in lister.run.call_args_list diff --git a/swh/lister/cgit/tests/test_tasks.py b/swh/lister/cgit/tests/test_tasks.py index 866bfde..c7d2fe3 100644 --- a/swh/lister/cgit/tests/test_tasks.py +++ b/swh/lister/cgit/tests/test_tasks.py @@ -1,28 +1,33 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.cgit.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.cgit.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.cgit.tasks.CGitLister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked CGitLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.cgit.tasks.CGitListerTask", kwargs=dict(url="https://git.kernel.org/", instance="kernel"), ) assert res res.wait() assert res.successful() lister.assert_called_once_with(url="https://git.kernel.org/", instance="kernel") lister.db_last_index.assert_not_called() lister.run.assert_called_once_with() diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/core/tests/conftest.py index 5e62987..21ce76f 100644 --- a/swh/lister/core/tests/conftest.py +++ b/swh/lister/core/tests/conftest.py @@ -1,53 +1,51 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.scheduler.tests.conftest import * # noqa - import logging import pytest from sqlalchemy import create_engine from swh.lister import get_lister, SUPPORTED_LISTERS from swh.lister.core.models import initialize logger = logging.getLogger(__name__) @pytest.fixture def lister_db_url(postgresql_proc, postgresql): db_url = "postgresql://{user}@{host}:{port}/{dbname}".format( host=postgresql_proc.host, port=postgresql_proc.port, user="postgres", dbname="tests", ) logger.debug("lister db_url: %s", db_url) return db_url @pytest.fixture def swh_listers(request, lister_db_url, swh_scheduler): listers = {} # Prepare schema for all listers for lister_name in SUPPORTED_LISTERS: lister = get_lister(lister_name, db_url=lister_db_url) lister.scheduler = swh_scheduler # inject scheduler fixture listers[lister_name] = lister initialize(create_engine(lister_db_url), drop_tables=True) # Add the load-archive-files expected by some listers (gnu, cran, ...) swh_scheduler.create_task_type( { "type": "load-archive-files", "description": "Load archive files.", "backend_name": "swh.loader.package.tasks.LoadArchive", "default_interval": "1 day", } ) return listers diff --git a/swh/lister/cran/tests/test_tasks.py b/swh/lister/cran/tests/test_tasks.py index 9ff3d9f..8148a0a 100644 --- a/swh/lister/cran/tests/test_tasks.py +++ b/swh/lister/cran/tests/test_tasks.py @@ -1,25 +1,30 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.cran.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.cran.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.cran.tasks.CRANLister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked CRANLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.cran.tasks.CRANListerTask") + res = swh_scheduler_celery_app.send_task("swh.lister.cran.tasks.CRANListerTask") assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with() diff --git a/swh/lister/debian/tests/test_tasks.py b/swh/lister/debian/tests/test_tasks.py index 4fb08bd..dbf1136 100644 --- a/swh/lister/debian/tests/test_tasks.py +++ b/swh/lister/debian/tests/test_tasks.py @@ -1,29 +1,31 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.debian.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.debian.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.debian.tasks.DebianLister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked DebianLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.debian.tasks.DebianListerTask", ("stretch",)) + res = swh_scheduler_celery_app.send_task( + "swh.lister.debian.tasks.DebianListerTask", ("stretch",) + ) assert res res.wait() assert res.successful() lister.assert_called_once_with(distribution="stretch") lister.run.assert_called_once_with() diff --git a/swh/lister/gitea/tests/test_tasks.py b/swh/lister/gitea/tests/test_tasks.py index 16352fa..4c223b6 100644 --- a/swh/lister/gitea/tests/test_tasks.py +++ b/swh/lister/gitea/tests/test_tasks.py @@ -1,141 +1,150 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from time import sleep from celery.result import GroupResult from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.gitea.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.gitea.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.gitea.tasks.GiteaLister") -def test_incremental(lister, swh_app, celery_session_worker): +def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GiteaLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 10, None) - res = swh_app.send_task("swh.lister.gitea.tasks.IncrementalGiteaLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.gitea.tasks.IncrementalGiteaLister" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with(sort="desc") lister.db_last_index.assert_not_called() lister.get_pages_information.assert_called_once_with() lister.run.assert_called_once_with(min_bound=1, max_bound=10, check_existence=True) @patch("swh.lister.gitea.tasks.GiteaLister") -def test_range(lister, swh_app, celery_session_worker): +def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GiteaLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.gitea.tasks.RangeGiteaLister", kwargs=dict(start=12, end=42) ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with(min_bound=12, max_bound=42) @patch("swh.lister.gitea.tasks.GiteaLister") -def test_relister(lister, swh_app, celery_session_worker): +def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GiteaLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 85, None) lister.db_partition_indices.return_value = [ (i, i + 9) for i in range(0, 80, 10) ] + [(80, 85)] - res = swh_app.send_task("swh.lister.gitea.tasks.FullGiteaRelister") + res = swh_scheduler_celery_app.send_task("swh.lister.gitea.tasks.FullGiteaRelister") assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with() # one by the FullGiteaRelister task # + 9 for the RangeGiteaLister subtasks assert lister.call_count == 10 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_not_called() lister.get_pages_information.assert_called_once_with() # lister.run should have been called once per partition interval for i in range(8): # XXX inconsistent behavior: max_bound is EXCLUDED here assert ( dict(min_bound=10 * i, max_bound=10 * i + 10), ) in lister.run.call_args_list assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list @patch("swh.lister.gitea.tasks.GiteaLister") -def test_relister_instance(lister, swh_app, celery_session_worker): +def test_relister_instance( + lister, swh_scheduler_celery_app, swh_scheduler_celery_worker +): # setup the mocked GiteaLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 85, None) lister.db_partition_indices.return_value = [ (i, i + 9) for i in range(0, 80, 10) ] + [(80, 85)] - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.gitea.tasks.FullGiteaRelister", kwargs=dict(url="https://0xacab.org/api/v4"), ) assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with(url="https://0xacab.org/api/v4") # one by the FullGiteaRelister task # + 9 for the RangeGiteaLister subtasks assert lister.call_count == 10 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_not_called() lister.get_pages_information.assert_called_once_with() # lister.run should have been called once per partition interval for i in range(8): # XXX inconsistent behavior: max_bound is EXCLUDED here assert ( dict(min_bound=10 * i, max_bound=10 * i + 10), ) in lister.run.call_args_list assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list diff --git a/swh/lister/github/tests/test_tasks.py b/swh/lister/github/tests/test_tasks.py index 721d88d..409ed6e 100644 --- a/swh/lister/github/tests/test_tasks.py +++ b/swh/lister/github/tests/test_tasks.py @@ -1,87 +1,91 @@ from time import sleep from celery.result import GroupResult from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.github.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.github.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.github.tasks.GitHubLister") -def test_incremental(lister, swh_app, celery_session_worker): +def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitHubLister lister.return_value = lister lister.db_last_index.return_value = 42 lister.run.return_value = None - res = swh_app.send_task("swh.lister.github.tasks.IncrementalGitHubLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.github.tasks.IncrementalGitHubLister" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_called_once_with() lister.run.assert_called_once_with(min_bound=42, max_bound=None) @patch("swh.lister.github.tasks.GitHubLister") -def test_range(lister, swh_app, celery_session_worker): +def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitHubLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.github.tasks.RangeGitHubLister", kwargs=dict(start=12, end=42) ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with(min_bound=12, max_bound=42) @patch("swh.lister.github.tasks.GitHubLister") -def test_relister(lister, swh_app, celery_session_worker): +def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitHubLister lister.return_value = lister lister.run.return_value = None lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)] - res = swh_app.send_task("swh.lister.github.tasks.FullGitHubRelister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.github.tasks.FullGitHubRelister" + ) assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with() # one by the FullGitHubRelister task # + 5 for the RangeGitHubLister subtasks assert lister.call_count == 6 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_called_once_with(10000) # lister.run should have been called once per partition interval for i in range(5): # XXX inconsistent behavior: max_bound is INCLUDED here assert ( dict(min_bound=10 * i, max_bound=10 * i + 9), ) in lister.run.call_args_list diff --git a/swh/lister/gitlab/tests/test_tasks.py b/swh/lister/gitlab/tests/test_tasks.py index 1959989..1e7f27e 100644 --- a/swh/lister/gitlab/tests/test_tasks.py +++ b/swh/lister/gitlab/tests/test_tasks.py @@ -1,141 +1,152 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from time import sleep from celery.result import GroupResult from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.gitlab.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.gitlab.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.gitlab.tasks.GitLabLister") -def test_incremental(lister, swh_app, celery_session_worker): +def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitlabLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 10, None) - res = swh_app.send_task("swh.lister.gitlab.tasks.IncrementalGitLabLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.gitlab.tasks.IncrementalGitLabLister" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with(sort="desc") lister.db_last_index.assert_not_called() lister.get_pages_information.assert_called_once_with() lister.run.assert_called_once_with(min_bound=1, max_bound=10, check_existence=True) @patch("swh.lister.gitlab.tasks.GitLabLister") -def test_range(lister, swh_app, celery_session_worker): +def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitlabLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.gitlab.tasks.RangeGitLabLister", kwargs=dict(start=12, end=42) ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with(min_bound=12, max_bound=42) @patch("swh.lister.gitlab.tasks.GitLabLister") -def test_relister(lister, swh_app, celery_session_worker): +def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GitlabLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 85, None) lister.db_partition_indices.return_value = [ (i, i + 9) for i in range(0, 80, 10) ] + [(80, 85)] - res = swh_app.send_task("swh.lister.gitlab.tasks.FullGitLabRelister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.gitlab.tasks.FullGitLabRelister" + ) assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with() # one by the FullGitlabRelister task # + 9 for the RangeGitlabLister subtasks assert lister.call_count == 10 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_not_called() lister.get_pages_information.assert_called_once_with() # lister.run should have been called once per partition interval for i in range(8): # XXX inconsistent behavior: max_bound is EXCLUDED here assert ( dict(min_bound=10 * i, max_bound=10 * i + 10), ) in lister.run.call_args_list assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list @patch("swh.lister.gitlab.tasks.GitLabLister") -def test_relister_instance(lister, swh_app, celery_session_worker): +def test_relister_instance( + lister, swh_scheduler_celery_app, swh_scheduler_celery_worker +): # setup the mocked GitlabLister lister.return_value = lister lister.run.return_value = None lister.get_pages_information.return_value = (None, 85, None) lister.db_partition_indices.return_value = [ (i, i + 9) for i in range(0, 80, 10) ] + [(80, 85)] - res = swh_app.send_task( + res = swh_scheduler_celery_app.send_task( "swh.lister.gitlab.tasks.FullGitLabRelister", kwargs=dict(url="https://0xacab.org/api/v4"), ) assert res res.wait() assert res.successful() # retrieve the GroupResult for this task and wait for all the subtasks # to complete promise_id = res.result assert promise_id - promise = GroupResult.restore(promise_id, app=swh_app) + promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) for i in range(5): if promise.ready(): break sleep(1) lister.assert_called_with(url="https://0xacab.org/api/v4") # one by the FullGitlabRelister task # + 9 for the RangeGitlabLister subtasks assert lister.call_count == 10 lister.db_last_index.assert_not_called() lister.db_partition_indices.assert_not_called() lister.get_pages_information.assert_called_once_with() # lister.run should have been called once per partition interval for i in range(8): # XXX inconsistent behavior: max_bound is EXCLUDED here assert ( dict(min_bound=10 * i, max_bound=10 * i + 10), ) in lister.run.call_args_list assert (dict(min_bound=80, max_bound=85),) in lister.run.call_args_list diff --git a/swh/lister/gnu/tests/test_tasks.py b/swh/lister/gnu/tests/test_tasks.py index d496798..d91f86a 100644 --- a/swh/lister/gnu/tests/test_tasks.py +++ b/swh/lister/gnu/tests/test_tasks.py @@ -1,25 +1,30 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.gnu.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.gnu.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.gnu.tasks.GNULister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked GNULister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.gnu.tasks.GNUListerTask") + res = swh_scheduler_celery_app.send_task("swh.lister.gnu.tasks.GNUListerTask") assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with() diff --git a/swh/lister/launchpad/tests/test_tasks.py b/swh/lister/launchpad/tests/test_tasks.py index ea676c5..4642373 100644 --- a/swh/lister/launchpad/tests/test_tasks.py +++ b/swh/lister/launchpad/tests/test_tasks.py @@ -1,33 +1,42 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from unittest.mock import patch @patch("swh.lister.launchpad.tasks.LaunchpadLister") -def test_new(lister, swh_app, celery_session_worker): +def test_new(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked LaunchpadLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.launchpad.tasks.NewLaunchpadLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.launchpad.tasks.NewLaunchpadLister" + ) assert res res.wait() assert res.successful() assert lister.call_count == 2 lister.db_last_threshold.assert_called_once() lister.run.assert_called_once() @patch("swh.lister.launchpad.tasks.LaunchpadLister") -def test_full(lister, swh_app, celery_session_worker): +def test_full(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked LaunchpadLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.launchpad.tasks.FullLaunchpadLister") + res = swh_scheduler_celery_app.send_task( + "swh.lister.launchpad.tasks.FullLaunchpadLister" + ) assert res res.wait() assert res.successful() lister.assert_called_once() lister.db_last_threshold.assert_not_called() lister.run.assert_called_once_with(max_bound=None) diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py index 382e557..481d91d 100644 --- a/swh/lister/npm/tests/test_tasks.py +++ b/swh/lister/npm/tests/test_tasks.py @@ -1,52 +1,61 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from contextlib import contextmanager from unittest.mock import patch @contextmanager def mock_save(lister): yield -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.npm.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.npm.tasks.save_registry_state") @patch("swh.lister.npm.tasks.NpmLister") -def test_lister(lister, save, swh_app, celery_session_worker): +def test_lister(lister, save, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked NpmLister lister.return_value = lister lister.run.return_value = None save.side_effect = mock_save - res = swh_app.send_task("swh.lister.npm.tasks.NpmListerTask") + res = swh_scheduler_celery_app.send_task("swh.lister.npm.tasks.NpmListerTask") assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.run.assert_called_once_with() @patch("swh.lister.npm.tasks.save_registry_state") @patch("swh.lister.npm.tasks.get_last_update_seq") @patch("swh.lister.npm.tasks.NpmIncrementalLister") -def test_incremental(lister, seq, save, swh_app, celery_session_worker): +def test_incremental( + lister, seq, save, swh_scheduler_celery_app, swh_scheduler_celery_worker +): # setup the mocked NpmLister lister.return_value = lister lister.run.return_value = None seq.return_value = 42 save.side_effect = mock_save - res = swh_app.send_task("swh.lister.npm.tasks.NpmIncrementalListerTask") + res = swh_scheduler_celery_app.send_task( + "swh.lister.npm.tasks.NpmIncrementalListerTask" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with() seq.assert_called_once_with(lister) lister.run.assert_called_once_with(min_bound=42) diff --git a/swh/lister/packagist/tests/test_tasks.py b/swh/lister/packagist/tests/test_tasks.py index 7c89b5b..6c5d15d 100644 --- a/swh/lister/packagist/tests/test_tasks.py +++ b/swh/lister/packagist/tests/test_tasks.py @@ -1,29 +1,31 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2020 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.packagist.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.packagist.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.packagist.tasks.PackagistLister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked PackagistLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.packagist.tasks.PackagistListerTask") + res = swh_scheduler_celery_app.send_task( + "swh.lister.packagist.tasks.PackagistListerTask" + ) assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with() diff --git a/swh/lister/phabricator/tests/test_tasks.py b/swh/lister/phabricator/tests/test_tasks.py index 38e1686..1aca578 100644 --- a/swh/lister/phabricator/tests/test_tasks.py +++ b/swh/lister/phabricator/tests/test_tasks.py @@ -1,11 +1,11 @@ -# Copyright (C) 2019 the Software Heritage developers +# Copyright (C) 2019-2020 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.phabricator.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.phabricator.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" diff --git a/swh/lister/pypi/tests/test_tasks.py b/swh/lister/pypi/tests/test_tasks.py index 89ffeac..2443cce 100644 --- a/swh/lister/pypi/tests/test_tasks.py +++ b/swh/lister/pypi/tests/test_tasks.py @@ -1,25 +1,30 @@ +# Copyright (C) 2019-2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + from unittest.mock import patch -def test_ping(swh_app, celery_session_worker): - res = swh_app.send_task("swh.lister.pypi.tasks.ping") +def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): + res = swh_scheduler_celery_app.send_task("swh.lister.pypi.tasks.ping") assert res res.wait() assert res.successful() assert res.result == "OK" @patch("swh.lister.pypi.tasks.PyPILister") -def test_lister(lister, swh_app, celery_session_worker): +def test_lister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): # setup the mocked PypiLister lister.return_value = lister lister.run.return_value = None - res = swh_app.send_task("swh.lister.pypi.tasks.PyPIListerTask") + res = swh_scheduler_celery_app.send_task("swh.lister.pypi.tasks.PyPIListerTask") assert res res.wait() assert res.successful() lister.assert_called_once_with() lister.db_last_index.assert_not_called() lister.run.assert_called_once_with() diff --git a/swh/lister/tests/test_cli.py b/swh/lister/tests/test_cli.py index 4ac7b1b..a333804 100644 --- a/swh/lister/tests/test_cli.py +++ b/swh/lister/tests/test_cli.py @@ -1,66 +1,66 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest from swh.lister.core.lister_base import ListerBase from swh.lister.cli import get_lister, SUPPORTED_LISTERS from .test_utils import init_db def test_get_lister_wrong_input(): """Unsupported lister should raise""" with pytest.raises(ValueError) as e: get_lister("unknown", "db-url") assert "Invalid lister" in str(e.value) def test_get_lister(): """Instantiating a supported lister should be ok """ db_url = init_db().url() for lister_name in SUPPORTED_LISTERS: lst = get_lister(lister_name, db_url) assert isinstance(lst, ListerBase) def test_get_lister_override(): """Overriding the lister configuration should populate its config """ db_url = init_db().url() listers = { "gitlab": "https://other.gitlab.uni/api/v4/", "phabricator": "https://somewhere.org/api/diffusion.repository.search", "cgit": "https://some.where/cgit", } # check the override ends up defined in the lister for lister_name, url in listers.items(): lst = get_lister( lister_name, db_url, **{"url": url, "priority": "high", "policy": "oneshot",}, ) assert lst.url == url assert lst.config["priority"] == "high" assert lst.config["policy"] == "oneshot" # check the default urls are used and not the override (since it's not # passed) for lister_name, url in listers.items(): lst = get_lister(lister_name, db_url) # no override so this does not end up in lister's configuration assert "url" not in lst.config assert "priority" not in lst.config assert "oneshot" not in lst.config assert lst.url == lst.DEFAULT_URL diff --git a/tox.ini b/tox.ini index ef77275..ff5b5ec 100644 --- a/tox.ini +++ b/tox.ini @@ -1,36 +1,37 @@ [tox] envlist=black,flake8,mypy,py3 [testenv] extras = testing deps = swh.core[http] >= 0.0.61 + swh.scheduler[testing] >= 0.5.0 pytest-cov dev: ipdb commands = pytest \ !dev: --cov={envsitepackagesdir}/swh/lister/ --cov-branch \ {envsitepackagesdir}/swh/lister/ {posargs} [testenv:black] skip_install = true deps = black commands = {envpython} -m black --check swh [testenv:flake8] skip_install = true deps = flake8 commands = {envpython} -m flake8 [testenv:mypy] extras = testing deps = mypy commands = mypy swh