diff --git a/PKG-INFO b/PKG-INFO index 0627851..2de3505 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,28 +1,28 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.0.67 +Version: 0.0.68 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh.scheduler.egg-info/PKG-INFO b/swh.scheduler.egg-info/PKG-INFO index 0627851..2de3505 100644 --- a/swh.scheduler.egg-info/PKG-INFO +++ b/swh.scheduler.egg-info/PKG-INFO @@ -1,28 +1,28 @@ Metadata-Version: 2.1 Name: swh.scheduler -Version: 0.0.67 +Version: 0.0.68 Summary: Software Heritage Scheduler Home-page: https://forge.softwareheritage.org/diffusion/DSCH/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-scheduler Description: swh-scheduler ============= Job scheduler for the Software Heritage project. Task manager for asynchronous/delayed tasks, used for both recurrent (e.g., listing a forge, loading new stuff from a Git repository) and one-off activities (e.g., loading a specific version of a source package). Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh/scheduler/tests/es/conftest.py b/swh/scheduler/tests/es/conftest.py index c91d110..f669cfd 100644 --- a/swh/scheduler/tests/es/conftest.py +++ b/swh/scheduler/tests/es/conftest.py @@ -1,53 +1,53 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import yaml import pytest from swh.scheduler import get_scheduler @pytest.fixture -def swh_scheduler_config(swh_scheduler_config): +def swh_scheduler_conf(swh_scheduler_config): return { 'scheduler': { 'cls': 'local', 'args': swh_scheduler_config, }, 'elasticsearch': { 'cls': 'memory', 'args': { 'index_name_prefix': 'swh-tasks', }, }, } @pytest.fixture -def swh_scheduler_config_file(swh_scheduler_config, monkeypatch, tmp_path): +def swh_scheduler_conf_file(swh_scheduler_conf, monkeypatch, tmp_path): conffile = str(tmp_path / 'elastic.yml') with open(conffile, 'w') as f: - f.write(yaml.dump(swh_scheduler_config)) + f.write(yaml.dump(swh_scheduler_conf)) monkeypatch.setenv('SWH_CONFIG_FILENAME', conffile) return conffile @pytest.fixture -def swh_scheduler(swh_scheduler_config): - return get_scheduler(**swh_scheduler_config['scheduler']) +def swh_scheduler(swh_scheduler_conf): + return get_scheduler(**swh_scheduler_conf['scheduler']) @pytest.fixture -def swh_elasticsearch(swh_scheduler_config): +def swh_elasticsearch(swh_scheduler_conf): from swh.scheduler.backend_es import ElasticSearchBackend - backend = ElasticSearchBackend(**swh_scheduler_config) + backend = ElasticSearchBackend(**swh_scheduler_conf) backend.initialize() return backend @pytest.fixture def swh_memory_elasticsearch(swh_elasticsearch): return swh_elasticsearch.storage diff --git a/swh/scheduler/tests/es/test_cli_task.py b/swh/scheduler/tests/es/test_cli_task.py index bbfefae..40b3ad0 100644 --- a/swh/scheduler/tests/es/test_cli_task.py +++ b/swh/scheduler/tests/es/test_cli_task.py @@ -1,108 +1,108 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import arrow import datetime import logging import uuid import random import pytest from click.testing import CliRunner from swh.scheduler.cli import cli from ..common import tasks_from_template, TASK_TYPES, TEMPLATES logger = logging.getLogger(__name__) @pytest.mark.usefixtures('swh_elasticsearch') -def test_cli_archive_tasks(swh_scheduler, swh_scheduler_config_file): +def test_cli_archive_tasks(swh_scheduler, swh_scheduler_conf_file): template_git = TEMPLATES['git'] template_hg = TEMPLATES['hg'] # first initialize scheduler's db (is this still needed?) for tt in TASK_TYPES.values(): swh_scheduler.create_task_type(tt) next_run_start = arrow.utcnow().datetime - datetime.timedelta(days=1) recurring = tasks_from_template( template_git, next_run_start, 100) oneshots = tasks_from_template( template_hg, next_run_start - datetime.timedelta(days=1), 50) past_time = next_run_start - datetime.timedelta(days=7) all_tasks = recurring + oneshots result = swh_scheduler.create_tasks(all_tasks) assert len(result) == len(all_tasks) # simulate task run backend_tasks = [ { 'task': task['id'], 'backend_id': str(uuid.uuid4()), 'scheduled': next_run_start - datetime.timedelta(minutes=i % 60), } for i, task in enumerate(result) ] swh_scheduler.mass_schedule_task_runs(backend_tasks) # Disable some tasks tasks_to_disable = set() for task in result: status = random.choice(['disabled', 'completed']) if status == 'disabled': tasks_to_disable.add(task['id']) swh_scheduler.disable_tasks(tasks_to_disable) git_tasks = swh_scheduler.search_tasks(task_type=template_git['type']) hg_tasks = swh_scheduler.search_tasks(task_type=template_hg['type']) assert len(git_tasks) + len(hg_tasks) == len(all_tasks) # Ensure the task_run are in expected state task_runs = swh_scheduler.get_task_runs([ t['id'] for t in git_tasks + hg_tasks ]) # Same for the tasks for t in git_tasks + hg_tasks: if t['id'] in tasks_to_disable: assert t['status'] == 'disabled' future_time = next_run_start + datetime.timedelta(days=1) for tr in task_runs: assert past_time <= tr['scheduled'] assert tr['scheduled'] < future_time runner = CliRunner() result = runner.invoke(cli, [ - '--config-file', swh_scheduler_config_file, + '--config-file', swh_scheduler_conf_file, 'task', 'archive', '--after', past_time.isoformat(), '--before', future_time.isoformat(), '--cleanup', ], obj={ 'log_level': logging.DEBUG, }) assert result.exit_code == 0, result.output # disabled tasks should no longer be in the scheduler git_tasks = swh_scheduler.search_tasks(task_type=template_git['type']) hg_tasks = swh_scheduler.search_tasks(task_type=template_hg['type']) remaining_tasks = git_tasks + hg_tasks count_disabled = 0 for task in remaining_tasks: logger.debug(f"task status: {task['status']}") if task['status'] == 'disabled': count_disabled += 1 assert count_disabled == 0 assert len(remaining_tasks) == len(all_tasks) - len(tasks_to_disable) diff --git a/version.txt b/version.txt index 9d31711..4d0c673 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.67-0-ge9d8a5f \ No newline at end of file +v0.0.68-0-g73ade78 \ No newline at end of file