diff --git a/swh/scheduler/simulator/origins.py b/swh/scheduler/simulator/origins.py --- a/swh/scheduler/simulator/origins.py +++ b/swh/scheduler/simulator/origins.py @@ -54,7 +54,7 @@ now = datetime.now(tz=timezone.utc) url = f"https://example.com/{_nb_generated_origins:06d}.git" - visit_type = "git" + visit_type = "test-git" origin = OriginModel(visit_type, url) return ListedOrigin( diff --git a/swh/scheduler/tests/common.py b/swh/scheduler/tests/common.py --- a/swh/scheduler/tests/common.py +++ b/swh/scheduler/tests/common.py @@ -8,13 +8,13 @@ from typing import Dict, List, Optional TEMPLATES = { - "git": { - "type": "load-git", + "test-git": { + "type": "load-test-git", "arguments": {"args": [], "kwargs": {},}, "next_run": None, }, - "hg": { - "type": "load-hg", + "test-hg": { + "type": "load-test-hg", "arguments": {"args": [], "kwargs": {},}, "next_run": None, "policy": "oneshot", @@ -23,8 +23,8 @@ TASK_TYPES = { - "git": { - "type": "load-git", + "test-git": { + "type": "load-test-git", "description": "Update a git repository", "backend_name": "swh.loader.git.tasks.UpdateGitRepository", "default_interval": datetime.timedelta(days=64), @@ -35,8 +35,8 @@ "num_retries": 7, "retry_delay": datetime.timedelta(hours=2), }, - "hg": { - "type": "load-hg", + "test-hg": { + "type": "load-test-hg", "description": "Update a mercurial repository", "backend_name": "swh.loader.mercurial.tasks.UpdateHgRepository", "default_interval": datetime.timedelta(days=64), diff --git a/swh/scheduler/tests/conftest.py b/swh/scheduler/tests/conftest.py --- a/swh/scheduler/tests/conftest.py +++ b/swh/scheduler/tests/conftest.py @@ -31,7 +31,7 @@ @pytest.fixture def visit_types() -> List[str]: """Possible visit types in `ListedOrigin`s""" - return ["git", "svn"] + return ["test-git", "test-svn"] @pytest.fixture diff --git a/swh/scheduler/tests/test_cli_origin.py b/swh/scheduler/tests/test_cli_origin.py --- a/swh/scheduler/tests/test_cli_origin.py +++ b/swh/scheduler/tests/test_cli_origin.py @@ -60,6 +60,8 @@ NUM_RESULTS = 10 # Strict inequality to check that grab_next_visits doesn't return more # results than requested + + # XXX: should test all of 'listed_origins_by_type' here... visit_type = next(iter(listed_origins_by_type)) assert len(listed_origins_by_type[visit_type]) > NUM_RESULTS diff --git a/swh/scheduler/tests/test_common.py b/swh/scheduler/tests/test_common.py --- a/swh/scheduler/tests/test_common.py +++ b/swh/scheduler/tests/test_common.py @@ -10,7 +10,7 @@ def test_tasks_from_template_no_priority(): nb_tasks = 3 - template = TEMPLATES["git"] + template = TEMPLATES["test-git"] next_run = datetime.datetime.utcnow() tasks = tasks_from_template(template, next_run, nb_tasks) @@ -27,7 +27,7 @@ def test_tasks_from_template_priority(): - template = TEMPLATES["hg"] + template = TEMPLATES["test-hg"] num_priorities = { None: 3, "high": 5, diff --git a/swh/scheduler/tests/test_recurrent_visits.py b/swh/scheduler/tests/test_recurrent_visits.py --- a/swh/scheduler/tests/test_recurrent_visits.py +++ b/swh/scheduler/tests/test_recurrent_visits.py @@ -6,7 +6,7 @@ from datetime import timedelta import logging from queue import Queue -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest @@ -34,7 +34,7 @@ @pytest.fixture def swh_scheduler(swh_scheduler): """Override default fixture of the scheduler to install some more task types.""" - for visit_type in ["git", "hg", "svn"]: + for visit_type in ["test-git", "test-hg", "test-svn"]: task_type = f"load-{visit_type}" swh_scheduler.create_task_type( { @@ -58,7 +58,13 @@ invoke( swh_scheduler, False, - ["schedule-recurrent", "--visit-type", "unknown", "--visit-type", "git"], + [ + "schedule-recurrent", + "--visit-type", + "unknown", + "--visit-type", + "test-git", + ], ) @@ -108,7 +114,7 @@ # we'll limit the orchestrator to the origins' type we know task_types.append(task_type) - for visit_type in ["git", "svn"]: + for visit_type in ["test-git", "test-svn"]: task_type = f"load-{visit_type}" send_visits_for_visit_type( swh_scheduler, mock_celery_app, visit_type, all_task_types[task_type] @@ -134,9 +140,12 @@ assert expected_record in set(records) +@patch.dict( + POLICY_ADDITIONAL_PARAMETERS, {"test-git": POLICY_ADDITIONAL_PARAMETERS["git"]} +) @pytest.mark.parametrize( "visit_type, tablesamples", - [("hg", {}), ("git", POLICY_ADDITIONAL_PARAMETERS["git"])], + [("test-hg", {}), ("test-git", POLICY_ADDITIONAL_PARAMETERS["git"])], ) def test_recurrent_visit_additional_parameters( swh_scheduler, mocker, visit_type, tablesamples diff --git a/swh/scheduler/tests/test_scheduler.py b/swh/scheduler/tests/test_scheduler.py --- a/swh/scheduler/tests/test_scheduler.py +++ b/swh/scheduler/tests/test_scheduler.py @@ -86,22 +86,22 @@ assert missing_methods == [] def test_add_task_type(self, swh_scheduler): - tt = TASK_TYPES["git"] + tt = TASK_TYPES["test-git"] swh_scheduler.create_task_type(tt) assert tt == swh_scheduler.get_task_type(tt["type"]) - tt2 = TASK_TYPES["hg"] + tt2 = TASK_TYPES["test-hg"] swh_scheduler.create_task_type(tt2) assert tt == swh_scheduler.get_task_type(tt["type"]) assert tt2 == swh_scheduler.get_task_type(tt2["type"]) def test_create_task_type_idempotence(self, swh_scheduler): - tt = TASK_TYPES["git"] + tt = TASK_TYPES["test-git"] swh_scheduler.create_task_type(tt) swh_scheduler.create_task_type(tt) assert tt == swh_scheduler.get_task_type(tt["type"]) def test_get_task_types(self, swh_scheduler): - tt, tt2 = TASK_TYPES["git"], TASK_TYPES["hg"] + tt, tt2 = TASK_TYPES["test-git"], TASK_TYPES["test-hg"] swh_scheduler.create_task_type(tt) swh_scheduler.create_task_type(tt2) actual_task_types = swh_scheduler.get_task_types() @@ -111,9 +111,9 @@ def test_create_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) num_git = 100 - tasks_1 = tasks_from_template(TEMPLATES["git"], utcnow(), num_git) + tasks_1 = tasks_from_template(TEMPLATES["test-git"], utcnow(), num_git) tasks_2 = tasks_from_template( - TEMPLATES["hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS + TEMPLATES["test-hg"], utcnow(), num_priorities=NUM_PRIORITY_TASKS ) tasks = tasks_1 + tasks_2 @@ -134,7 +134,7 @@ for task, orig_task in zip(ret, tasks): task = copy.deepcopy(task) - task_type = TASK_TYPES[orig_task["type"].split("-")[-1]] + task_type = TASK_TYPES[orig_task["type"].split("-", 1)[-1]] assert task["id"] not in ids assert task["status"] == "next_run_not_scheduled" assert task["current_interval"] == task_type["default_interval"] @@ -161,8 +161,8 @@ def test_peek_ready_tasks_no_priority(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + task_type = TEMPLATES["test-git"]["type"] + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) random.shuffle(tasks) swh_scheduler.create_tasks(tasks) @@ -203,10 +203,10 @@ """Peek ready tasks only return standard tasks (no priority)""" self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] # Create tasks with and without priorities tasks = tasks_from_template( - TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS, + TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS, ) count_priority = 0 @@ -230,10 +230,10 @@ def test_grab_ready_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] # Create tasks with and without priorities tasks = tasks_from_template( - TEMPLATES["git"], t, num_priorities=NUM_PRIORITY_TASKS + TEMPLATES["test-git"], t, num_priorities=NUM_PRIORITY_TASKS ) random.shuffle(tasks) swh_scheduler.create_tasks(tasks) @@ -257,17 +257,17 @@ """check the grab and peek priority tasks endpoint behave as expected""" self._create_task_types(swh_scheduler) t = utcnow() - task_type = TEMPLATES["git"]["type"] + task_type = TEMPLATES["test-git"]["type"] num_tasks = 100 # Create tasks with and without priorities tasks0 = tasks_with_priority_from_template( - TEMPLATES["git"], t, num_tasks, "high", + TEMPLATES["test-git"], t, num_tasks, "high", ) tasks1 = tasks_with_priority_from_template( - TEMPLATES["hg"], t, num_tasks, "low", + TEMPLATES["test-hg"], t, num_tasks, "low", ) tasks2 = tasks_with_priority_from_template( - TEMPLATES["hg"], t, num_tasks, "normal", + TEMPLATES["test-hg"], t, num_tasks, "normal", ) tasks = tasks0 + tasks1 + tasks2 @@ -291,7 +291,7 @@ def test_get_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) t = utcnow() - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) tasks = swh_scheduler.create_tasks(tasks) random.shuffle(tasks) while len(tasks) > 1: @@ -311,7 +311,7 @@ self._create_task_types(swh_scheduler) t = utcnow() - tasks = tasks_from_template(TEMPLATES["git"], t, 100) + tasks = tasks_from_template(TEMPLATES["test-git"], t, 100) tasks = swh_scheduler.create_tasks(tasks) assert make_real_dicts(swh_scheduler.search_tasks()) == make_real_dicts(tasks) @@ -336,8 +336,8 @@ """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) # simulate scheduling tasks @@ -458,8 +458,8 @@ def test_delete_archived_tasks(self, swh_scheduler): self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ @@ -505,8 +505,8 @@ """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) swh_scheduler.create_tasks(recurring + oneshots) assert not swh_scheduler.get_task_runs(task_ids=()) @@ -520,8 +520,8 @@ """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) total_tasks = len(recurring) + len(oneshots) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ @@ -574,8 +574,8 @@ """ self._create_task_types(swh_scheduler) _time = utcnow() - recurring = tasks_from_template(TEMPLATES["git"], _time, 12) - oneshots = tasks_from_template(TEMPLATES["hg"], _time, 12) + recurring = tasks_from_template(TEMPLATES["test-git"], _time, 12) + oneshots = tasks_from_template(TEMPLATES["test-hg"], _time, 12) pending_tasks = swh_scheduler.create_tasks(recurring + oneshots) backend_tasks = [ {