diff --git a/swh/lister/bitbucket/tasks.py b/swh/lister/bitbucket/tasks.py --- a/swh/lister/bitbucket/tasks.py +++ b/swh/lister/bitbucket/tasks.py @@ -45,9 +45,11 @@ lister = new_lister(**lister_args) ranges = lister.db_partition_indices(split or GROUP_SPLIT) random.shuffle(ranges) - group(range_bitbucket_lister.s(minv, maxv, **lister_args) - for minv, maxv in ranges)() + promise = group(range_bitbucket_lister.s(minv, maxv, **lister_args) + for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) + promise.save() # so that we can restore the GroupResult in tests + return promise.id @app.task(name='swh.lister.bitbucket.tasks.ping', diff --git a/swh/lister/bitbucket/tests/conftest.py b/swh/lister/bitbucket/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/bitbucket/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/bitbucket/tests/test_tasks.py b/swh/lister/bitbucket/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/bitbucket/tests/test_tasks.py @@ -0,0 +1,89 @@ +from time import sleep +from celery.result import GroupResult + +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.bitbucket.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.bitbucket.tasks.BitBucketLister') +def test_incremental(lister, swh_app, celery_session_worker): + # setup the mocked BitbucketLister + lister.return_value = lister + lister.db_last_index.return_value = 42 + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.bitbucket.tasks.IncrementalBitBucketLister') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0') + lister.db_last_index.assert_called_once_with() + lister.run.assert_called_once_with(min_bound=42, max_bound=None) + + +@patch('swh.lister.bitbucket.tasks.BitBucketLister') +def test_range(lister, swh_app, celery_session_worker): + # setup the mocked BitbucketLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.bitbucket.tasks.RangeBitBucketLister', + kwargs=dict(start=12, end=42)) + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with(api_baseurl='https://api.bitbucket.org/2.0') + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with(min_bound=12, max_bound=42) + + +@patch('swh.lister.bitbucket.tasks.BitBucketLister') +def test_relister(lister, swh_app, celery_session_worker): + # setup the mocked BitbucketLister + lister.return_value = lister + lister.run.return_value = None + lister.db_partition_indices.return_value = [ + (i, i+9) for i in range(0, 50, 10)] + + res = swh_app.send_task( + 'swh.lister.bitbucket.tasks.FullBitBucketRelister') + assert res + + res.wait() + assert res.successful() + + # retrieve the GroupResult for this task and wait for all the subtasks + # to complete + promise_id = res.result + assert promise_id + promise = GroupResult.restore(promise_id, app=swh_app) + for i in range(5): + if promise.ready(): + break + sleep(1) + + lister.assert_called_with(api_baseurl='https://api.bitbucket.org/2.0') + + # one by the FullBitbucketRelister task + # + 5 for the RangeBitbucketLister subtasks + assert lister.call_count == 6 + + lister.db_last_index.assert_not_called() + lister.db_partition_indices.assert_called_once_with(10000) + + # lister.run should have been called once per partition interval + for i in range(5): + assert (dict(min_bound=10*i, max_bound=10*i + 9),) \ + in lister.run.call_args_list diff --git a/swh/lister/core/tests/conftest.py b/swh/lister/core/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/core/tests/conftest.py @@ -0,0 +1,28 @@ +import pytest + + +@pytest.fixture(scope='session') +def celery_enable_logging(): + return True + + +@pytest.fixture(scope='session') +def celery_includes(): + return [ + 'swh.lister.bitbucket.tasks', + 'swh.lister.debian.tasks', + 'swh.lister.github.tasks', + 'swh.lister.gitlab.tasks', + 'swh.lister.npm.tasks', + 'swh.lister.pypi.tasks', + ] + + +# override the celery_session_app fixture to monkeypatch the 'main' +# swh.scheduler.celery_backend.config.app Celery application +# with the test application. +@pytest.fixture(scope='session') +def swh_app(celery_session_app): + import swh.scheduler.celery_backend.config + swh.scheduler.celery_backend.config.app = celery_session_app + yield celery_session_app diff --git a/swh/lister/debian/tests/__init__.py b/swh/lister/debian/tests/__init__.py new file mode 100644 diff --git a/swh/lister/debian/tests/conftest.py b/swh/lister/debian/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/debian/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/debian/tests/test_tasks.py b/swh/lister/debian/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/debian/tests/test_tasks.py @@ -0,0 +1,29 @@ +from time import sleep +from celery.result import GroupResult + +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.debian.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.debian.tasks.DebianLister') +def test_lister(lister, swh_app, celery_session_worker): + # setup the mocked DebianLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.debian.tasks.DebianListerTask', ('stretch',)) + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + lister.run.assert_called_once_with('stretch') diff --git a/swh/lister/github/tasks.py b/swh/lister/github/tasks.py --- a/swh/lister/github/tasks.py +++ b/swh/lister/github/tasks.py @@ -46,9 +46,11 @@ lister = new_lister(**lister_args) ranges = lister.db_partition_indices(split or GROUP_SPLIT) random.shuffle(ranges) - group(range_github_lister.s(minv, maxv, **lister_args) - for minv, maxv in ranges)() + promise = group(range_github_lister.s(minv, maxv, **lister_args) + for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) + promise.save() # so that we can restore the GroupResult in tests + return promise.id @app.task(name='swh.lister.github.tasks.ping', diff --git a/swh/lister/github/tests/conftest.py b/swh/lister/github/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/github/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/github/tests/test_tasks.py b/swh/lister/github/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/github/tests/test_tasks.py @@ -0,0 +1,90 @@ +from time import sleep +from celery.result import GroupResult + +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.github.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.github.tasks.GitHubLister') +def test_incremental(lister, swh_app, celery_session_worker): + # setup the mocked GitHubLister + lister.return_value = lister + lister.db_last_index.return_value = 42 + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.github.tasks.IncrementalGitHubLister') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with(api_baseurl='https://api.github.com') + lister.db_last_index.assert_called_once_with() + lister.run.assert_called_once_with(min_bound=42, max_bound=None) + + +@patch('swh.lister.github.tasks.GitHubLister') +def test_range(lister, swh_app, celery_session_worker): + # setup the mocked GitHubLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.github.tasks.RangeGitHubLister', + kwargs=dict(start=12, end=42)) + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with(api_baseurl='https://api.github.com') + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with(min_bound=12, max_bound=42) + + +@patch('swh.lister.github.tasks.GitHubLister') +def test_relister(lister, swh_app, celery_session_worker): + # setup the mocked GitHubLister + lister.return_value = lister + lister.run.return_value = None + lister.db_partition_indices.return_value = [ + (i, i+9) for i in range(0, 50, 10)] + + res = swh_app.send_task( + 'swh.lister.github.tasks.FullGitHubRelister') + assert res + + res.wait() + assert res.successful() + + # retrieve the GroupResult for this task and wait for all the subtasks + # to complete + promise_id = res.result + assert promise_id + promise = GroupResult.restore(promise_id, app=swh_app) + for i in range(5): + if promise.ready(): + break + sleep(1) + + lister.assert_called_with(api_baseurl='https://api.github.com') + + # one by the FullGitHubRelister task + # + 5 for the RangeGitHubLister subtasks + assert lister.call_count == 6 + + lister.db_last_index.assert_not_called() + lister.db_partition_indices.assert_called_once_with(10000) + + # lister.run should have been called once per partition interval + for i in range(5): + # XXX inconsistent behavior: max_bound is INCLUDED here + assert (dict(min_bound=10*i, max_bound=10*i + 9),) \ + in lister.run.call_args_list diff --git a/swh/lister/gitlab/tasks.py b/swh/lister/gitlab/tasks.py --- a/swh/lister/gitlab/tasks.py +++ b/swh/lister/gitlab/tasks.py @@ -54,9 +54,11 @@ _, total_pages, _ = lister.get_pages_information() ranges = list(utils.split_range(total_pages, NBPAGES)) random.shuffle(ranges) - group(range_gitlab_lister.s(minv, maxv, **lister_args) - for minv, maxv in ranges)() + promise = group(range_gitlab_lister.s(minv, maxv, **lister_args) + for minv, maxv in ranges)() self.log.debug('%s OK (spawned %s subtasks)' % (self.name, len(ranges))) + promise.save() + return promise.id @app.task(name='swh.lister.gitlab.tasks.ping', diff --git a/swh/lister/gitlab/tests/conftest.py b/swh/lister/gitlab/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/gitlab/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/gitlab/tests/test_tasks.py b/swh/lister/gitlab/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/gitlab/tests/test_tasks.py @@ -0,0 +1,102 @@ +from time import sleep +from celery.result import GroupResult + +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.gitlab.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.gitlab.tasks.GitLabLister') +def test_incremental(lister, swh_app, celery_session_worker): + # setup the mocked GitlabLister + lister.return_value = lister + lister.run.return_value = None + lister.get_pages_information.return_value = (None, 10, None) + + res = swh_app.send_task( + 'swh.lister.gitlab.tasks.IncrementalGitLabLister') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with( + api_baseurl='https://gitlab.com/api/v4', + instance='gitlab', sort='desc', per_page=20) + lister.db_last_index.assert_not_called() + lister.get_pages_information.assert_called_once_with() + lister.run.assert_called_once_with( + min_bound=1, max_bound=10, check_existence=True) + + +@patch('swh.lister.gitlab.tasks.GitLabLister') +def test_range(lister, swh_app, celery_session_worker): + # setup the mocked GitlabLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.gitlab.tasks.RangeGitLabLister', + kwargs=dict(start=12, end=42)) + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with( + api_baseurl='https://gitlab.com/api/v4', + instance='gitlab', sort='asc', per_page=20) + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with(min_bound=12, max_bound=42) + + +@patch('swh.lister.gitlab.tasks.GitLabLister') +def test_relister(lister, swh_app, celery_session_worker): + # setup the mocked GitlabLister + lister.return_value = lister + lister.run.return_value = None + lister.get_pages_information.return_value = (None, 85, None) + lister.db_partition_indices.return_value = [ + (i, i+9) for i in range(0, 80, 10)] + [(80, 85)] + + res = swh_app.send_task( + 'swh.lister.gitlab.tasks.FullGitLabRelister') + assert res + + res.wait() + assert res.successful() + + # retrieve the GroupResult for this task and wait for all the subtasks + # to complete + promise_id = res.result + assert promise_id + promise = GroupResult.restore(promise_id, app=swh_app) + for i in range(5): + if promise.ready(): + break + sleep(1) + + lister.assert_called_with( + api_baseurl='https://gitlab.com/api/v4', + instance='gitlab', sort='asc', per_page=20) + + # one by the FullGitlabRelister task + # + 9 for the RangeGitlabLister subtasks + assert lister.call_count == 10 + + lister.db_last_index.assert_not_called() + lister.db_partition_indices.assert_not_called() + lister.get_pages_information.assert_called_once_with() + + # lister.run should have been called once per partition interval + for i in range(8): + # XXX inconsistent behavior: max_bound is EXCLUDED here + assert (dict(min_bound=10*i, max_bound=10*i + 10),) \ + in lister.run.call_args_list + assert (dict(min_bound=80, max_bound=85),) \ + in lister.run.call_args_list diff --git a/swh/lister/npm/tests/conftest.py b/swh/lister/npm/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/npm/tests/test_tasks.py b/swh/lister/npm/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/npm/tests/test_tasks.py @@ -0,0 +1,55 @@ +from contextlib import contextmanager +from unittest.mock import patch + + +@contextmanager +def mock_save(lister): + yield + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.npm.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.npm.tasks.save_registry_state') +@patch('swh.lister.npm.tasks.NpmLister') +def test_lister(lister, save, swh_app, celery_session_worker): + # setup the mocked NpmLister + lister.return_value = lister + lister.run.return_value = None + save.side_effect = mock_save + + res = swh_app.send_task('swh.lister.npm.tasks.NpmListerTask') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + lister.run.assert_called_once_with() + + +@patch('swh.lister.npm.tasks.save_registry_state') +@patch('swh.lister.npm.tasks.get_last_update_seq') +@patch('swh.lister.npm.tasks.NpmIncrementalLister') +def test_incremental(lister, seq, save, swh_app, celery_session_worker): + # setup the mocked NpmLister + lister.return_value = lister + lister.run.return_value = None + lister.request_headers.return_value = [] + seq.return_value = 42 + save.side_effect = mock_save + + res = swh_app.send_task( + 'swh.lister.npm.tasks.NpmIncrementalListerTask') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + seq.assert_called_once_with(lister) + lister.run.assert_called_once_with(min_bound=42) diff --git a/swh/lister/pypi/tests/__init__.py b/swh/lister/pypi/tests/__init__.py new file mode 100644 diff --git a/swh/lister/pypi/tests/conftest.py b/swh/lister/pypi/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/lister/pypi/tests/conftest.py @@ -0,0 +1 @@ +from swh.lister.core.tests.conftest import * # noqa diff --git a/swh/lister/pypi/tests/test_tasks.py b/swh/lister/pypi/tests/test_tasks.py new file mode 100644 --- /dev/null +++ b/swh/lister/pypi/tests/test_tasks.py @@ -0,0 +1,27 @@ +from unittest.mock import patch + + +def test_ping(swh_app, celery_session_worker): + res = swh_app.send_task( + 'swh.lister.pypi.tasks.ping') + assert res + res.wait() + assert res.successful() + assert res.result == 'OK' + + +@patch('swh.lister.pypi.tasks.PyPILister') +def test_lister(lister, swh_app, celery_session_worker): + # setup the mocked PypiLister + lister.return_value = lister + lister.run.return_value = None + + res = swh_app.send_task( + 'swh.lister.pypi.tasks.PyPIListerTask') + assert res + res.wait() + assert res.successful() + + lister.assert_called_once_with() + lister.db_last_index.assert_not_called() + lister.run.assert_called_once_with()