Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/bitbucket/tests/test_tasks.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from time import sleep | |||||
from unittest.mock import patch | from unittest.mock import patch | ||||
from celery.result import GroupResult | from swh.lister.pattern import ListerStats | ||||
def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): | def test_ping(swh_scheduler_celery_app, swh_scheduler_celery_worker): | ||||
res = swh_scheduler_celery_app.send_task("swh.lister.bitbucket.tasks.ping") | res = swh_scheduler_celery_app.send_task("swh.lister.bitbucket.tasks.ping") | ||||
assert res | assert res | ||||
res.wait() | res.wait() | ||||
assert res.successful() | assert res.successful() | ||||
assert res.result == "OK" | assert res.result == "OK" | ||||
@patch("swh.lister.bitbucket.tasks.BitBucketLister") | @patch("swh.lister.bitbucket.tasks.BitbucketLister") | ||||
def test_incremental(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): | def test_incremental_listing( | ||||
# setup the mocked BitbucketLister | lister, swh_scheduler_celery_app, swh_scheduler_celery_worker | ||||
lister.return_value = lister | ): | ||||
lister.db_last_index.return_value = 42 | lister.from_configfile.return_value = lister | ||||
lister.run.return_value = None | lister.run.return_value = ListerStats(pages=5, origins=5000) | ||||
res = swh_scheduler_celery_app.send_task( | res = swh_scheduler_celery_app.send_task( | ||||
"swh.lister.bitbucket.tasks.IncrementalBitBucketLister" | "swh.lister.bitbucket.tasks.IncrementalBitBucketLister", | ||||
kwargs=dict(page_size=100, username="username", password="password",), | |||||
) | ) | ||||
assert res | assert res | ||||
res.wait() | res.wait() | ||||
assert res.successful() | assert res.successful() | ||||
lister.assert_called_once_with() | lister.run.assert_called_once() | ||||
lister.db_last_index.assert_called_once_with() | |||||
lister.run.assert_called_once_with(min_bound=42, max_bound=None) | |||||
@patch("swh.lister.bitbucket.tasks.BitBucketLister") | @patch("swh.lister.bitbucket.tasks.BitbucketLister") | ||||
def test_range(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): | def test_full_listing(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): | ||||
# setup the mocked BitbucketLister | lister.from_configfile.return_value = lister | ||||
lister.return_value = lister | lister.run.return_value = ListerStats(pages=5, origins=5000) | ||||
lister.run.return_value = None | |||||
res = swh_scheduler_celery_app.send_task( | res = swh_scheduler_celery_app.send_task( | ||||
"swh.lister.bitbucket.tasks.RangeBitBucketLister", kwargs=dict(start=12, end=42) | "swh.lister.bitbucket.tasks.FullBitBucketRelister", | ||||
kwargs=dict(page_size=100, username="username", password="password",), | |||||
) | ) | ||||
assert res | assert res | ||||
res.wait() | res.wait() | ||||
assert res.successful() | assert res.successful() | ||||
lister.assert_called_once_with() | lister.run.assert_called_once() | ||||
lister.db_last_index.assert_not_called() | |||||
lister.run.assert_called_once_with(min_bound=12, max_bound=42) | |||||
@patch("swh.lister.bitbucket.tasks.BitBucketLister") | |||||
def test_relister(lister, swh_scheduler_celery_app, swh_scheduler_celery_worker): | |||||
# setup the mocked BitbucketLister | |||||
lister.return_value = lister | |||||
lister.run.return_value = None | |||||
lister.db_partition_indices.return_value = [(i, i + 9) for i in range(0, 50, 10)] | |||||
res = swh_scheduler_celery_app.send_task( | |||||
"swh.lister.bitbucket.tasks.FullBitBucketRelister" | |||||
) | |||||
assert res | |||||
res.wait() | |||||
assert res.successful() | |||||
# retrieve the GroupResult for this task and wait for all the subtasks | |||||
# to complete | |||||
promise_id = res.result | |||||
assert promise_id | |||||
promise = GroupResult.restore(promise_id, app=swh_scheduler_celery_app) | |||||
for i in range(5): | |||||
if promise.ready(): | |||||
break | |||||
sleep(1) | |||||
lister.assert_called_with() | |||||
# one by the FullBitbucketRelister task | |||||
# + 5 for the RangeBitbucketLister subtasks | |||||
assert lister.call_count == 6 | |||||
lister.db_last_index.assert_not_called() | |||||
lister.db_partition_indices.assert_called_once_with(10000) | |||||
# lister.run should have been called once per partition interval | |||||
for i in range(5): | |||||
assert ( | |||||
dict(min_bound=10 * i, max_bound=10 * i + 9), | |||||
) in lister.run.call_args_list |