diff --git a/conftest.py b/conftest.py --- a/conftest.py +++ b/conftest.py @@ -4,3 +4,13 @@ # https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles settings.register_profile("fast", max_examples=5, deadline=5000) settings.register_profile("slow", max_examples=20, deadline=5000) + +# Modules that should not be loaded by --doctest-modules +collect_ignore = [ + # ImportError + 'swh/scheduler/updater/ghtorrent/fake.py', + # NotImplementedError: save_group is not supported by this backend. + 'swh/scheduler/tests/tasks.py', + # OSError: Configuration file must be defined + 'swh/scheduler/api/wsgi.py', +] diff --git a/swh/scheduler/cli.py b/swh/scheduler/cli.py --- a/swh/scheduler/cli.py +++ b/swh/scheduler/cli.py @@ -70,6 +70,46 @@ """Pretty-print a task If 'full' is True, also print the status and priority fields. + + >>> task = { + ... 'id': 1234, + ... 'arguments': { + ... 'args': ['foo', 'bar'], + ... 'kwargs': {'key': 'value'}, + ... }, + ... 'current_interval': datetime.timedelta(hours=1), + ... 'next_run': datetime.datetime(2019, 2, 21, 13, 52, 35, 407818), + ... 'policy': 'oneshot', + ... 'priority': None, + ... 'status': 'next_run_not_scheduled', + ... 'type': 'test_task', + ... } + >>> print(click.unstyle(pretty_print_task(task))) + Task 1234 + Next run: ... (2019-02-21 13:52:35+00:00) + Interval: 1:00:00 + Type: test_task + Policy: oneshot + Args: + foo + bar + Keyword args: + key: value + + >>> print(click.unstyle(pretty_print_task(task, full=True))) + Task 1234 + Next run: ... (2019-02-21 13:52:35+00:00) + Interval: 1:00:00 + Type: test_task + Policy: oneshot + Status: next_run_not_scheduled + Priority:\x20 + Args: + foo + bar + Keyword args: + key: value + """ next_run = arrow.get(task['next_run']) lines = [ diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py --- a/swh/scheduler/tests/conftest.py +++ b/swh/scheduler/tests/conftest.py @@ -20,7 +20,7 @@ DUMP_FILES = os.path.join(SQL_DIR, '*.sql') # celery tasks for testing purpose; tasks themselves should be -# in swh/scheduler/tests/celery_tasks.py +# in swh/scheduler/tests/tasks.py TASK_NAMES = ['ping', 'multiping', 'add', 'error'] diff --git a/swh/scheduler/tests/test_cli.py b/swh/scheduler/tests/test_cli.py new file mode 100644 --- /dev/null +++ b/swh/scheduler/tests/test_cli.py @@ -0,0 +1,191 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import datetime +import re +import tempfile +from unittest.mock import patch + +from click.testing import CliRunner + +from swh.scheduler.cli import cli +from swh.scheduler.utils import create_task_dict + + +CLI_CONFIG = ''' +scheduler: + cls: foo + args: {} +''' + + +def invoke(scheduler, catch_exceptions, args): + runner = CliRunner() + with patch('swh.scheduler.cli.get_scheduler') as get_scheduler_mock, \ + tempfile.NamedTemporaryFile('a', suffix='.yml') as config_fd: + config_fd.write(CLI_CONFIG) + config_fd.seek(0) + get_scheduler_mock.return_value = scheduler + result = runner.invoke(cli, ['-C' + config_fd.name] + args) + if not catch_exceptions and result.exception: + print(result.output) + raise result.exception + return result + + +def test_schedule_tasks(swh_scheduler): + csv_data = ( + b'swh-test-ping;[["arg1", "arg2"]];{"key": "value"};' + + datetime.datetime.utcnow().isoformat().encode() + b'\n' + + b'swh-test-ping;[["arg3", "arg4"]];{"key": "value"};' + + datetime.datetime.utcnow().isoformat().encode() + b'\n') + with tempfile.NamedTemporaryFile(suffix='.csv') as csv_fd: + csv_fd.write(csv_data) + csv_fd.seek(0) + result = invoke(swh_scheduler, False, [ + 'task', 'schedule', + '-d', ';', + csv_fd.name + ]) + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Created 2 tasks + +Task 1 + Next run: just now \(.*\) + Interval: 1 day, 0:00:00 + Type: swh-test-ping + Policy: recurring + Args: + \['arg1', 'arg2'\] + Keyword args: + key: value + +Task 2 + Next run: just now \(.*\) + Interval: 1 day, 0:00:00 + Type: swh-test-ping + Policy: recurring + Args: + \['arg3', 'arg4'\] + Keyword args: + key: value + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output + + +def test_schedule_tasks_columns(swh_scheduler): + with tempfile.NamedTemporaryFile(suffix='.csv') as csv_fd: + csv_fd.write( + b'swh-test-ping;oneshot;["arg1", "arg2"];{"key": "value"}\n') + csv_fd.seek(0) + result = invoke(swh_scheduler, False, [ + 'task', 'schedule', + '-c', 'type', '-c', 'policy', '-c', 'args', '-c', 'kwargs', + '-d', ';', + csv_fd.name + ]) + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Created 1 tasks + +Task 1 + Next run: just now \(.*\) + Interval: 1 day, 0:00:00 + Type: swh-test-ping + Policy: oneshot + Args: + arg1 + arg2 + Keyword args: + key: value + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output + + +def test_schedule_task(swh_scheduler): + result = invoke(swh_scheduler, False, [ + 'task', 'add', + 'swh-test-ping', 'arg1', 'arg2', 'key=value', + ]) + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Created 1 tasks + +Task 1 + Next run: just now \(.*\) + Interval: 1 day, 0:00:00 + Type: swh-test-ping + Policy: recurring + Args: + arg1 + arg2 + Keyword args: + key: value + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output + + +def test_list_pending_tasks_none(swh_scheduler): + result = invoke(swh_scheduler, False, [ + 'task', 'list-pending', 'swh-test-ping', + ]) + + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Found 0 swh-test-ping tasks + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output + + +def test_list_pending_tasks_one(swh_scheduler): + task = create_task_dict('swh-test-ping', 'oneshot', key='value') + swh_scheduler.create_tasks([task]) + + result = invoke(swh_scheduler, False, [ + 'task', 'list-pending', 'swh-test-ping', + ]) + + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Found 1 swh-test-ping tasks + +Task 1 + Next run: just now \(.*\) + Interval: 1 day, 0:00:00 + Type: swh-test-ping + Policy: oneshot + Args: + Keyword args: + key: value + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output + + +def test_list_pending_tasks_one_filter(swh_scheduler): + task = create_task_dict('swh-test-multiping', 'oneshot', key='value') + swh_scheduler.create_tasks([task]) + + result = invoke(swh_scheduler, False, [ + 'task', 'list-pending', 'swh-test-ping', + ]) + + expected = r''' +\[INFO\] swh.core.config -- Loading config file .* +Found 0 swh-test-ping tasks + +'''.lstrip() + assert result.exit_code == 0, result.output + assert re.fullmatch(expected, result.output, re.MULTILINE), result.output diff --git a/tox.ini b/tox.ini --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ LC_CTYPE=C.UTF-8 LANG=C.UTF-8 commands = - pifpaf run postgresql -- pytest --hypothesis-profile=fast --cov=swh --cov-branch {posargs} + pifpaf run postgresql -- pytest --doctest-modules --hypothesis-profile=fast --cov=swh --cov-branch {posargs} [testenv:py3-slow] deps = @@ -19,7 +19,7 @@ pytest-cov pifpaf commands = - pifpaf run postgresql -- pytest --hypothesis-profile=slow --cov=swh --cov-branch {posargs} + pifpaf run postgresql -- pytest --doctest-modules --hypothesis-profile=slow --cov=swh --cov-branch {posargs} [testenv:flake8]