diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -110,6 +110,17 @@ "disabled": SAVE_TASK_FAILED, } +# map scheduler task_run status to origin save status +_save_task_run_status = { + "scheduled": SAVE_TASK_SCHEDULED, + "started": SAVE_TASK_RUNNING, + "eventful": SAVE_TASK_SUCCEEDED, + "uneventful": SAVE_TASK_SUCCEEDED, + "failed": SAVE_TASK_FAILED, + "permfailed": SAVE_TASK_FAILED, + "lost": SAVE_TASK_FAILED, +} + def get_savable_visit_types(): return sorted(list(_visit_type_task.keys())) @@ -186,12 +197,14 @@ return visit_date, save_task_status -def _save_request_dict(save_request, task=None): +def _save_request_dict(save_request, task=None, task_run=None): must_save = False visit_date = save_request.visit_date # save task still in scheduler db if task: save_task_status = _save_task_status[task["status"]] + if task_run: + save_task_status = _save_task_run_status[task_run["status"]] # Consider request from which a visit date has already been found # as succeeded to avoid retrieving it again if save_task_status == SAVE_TASK_SCHEDULED and visit_date: @@ -313,7 +326,10 @@ # get the scheduler task and its status tasks = scheduler.get_tasks([sor.loading_task_id]) task = tasks[0] if tasks else None - task_status = _save_request_dict(sor, task)["save_task_status"] + task_runs = scheduler.get_task_runs([sor.loading_task_id]) + task_run = task_runs[0] if task_runs else None + save_request = _save_request_dict(sor, task, task_run) + task_status = save_request["save_task_status"] # create a new scheduler task only if the previous one has been # already executed if ( @@ -392,8 +408,12 @@ if task_ids: tasks = scheduler.get_tasks(task_ids) tasks = {task["id"]: task for task in tasks} + task_runs = scheduler.get_task_runs(tasks) + task_runs = {task_run["task"]: task_run for task_run in task_runs} for sor in requests_queryset: - sr_dict = _save_request_dict(sor, tasks.get(sor.loading_task_id)) + sr_dict = _save_request_dict( + sor, tasks.get(sor.loading_task_id), task_runs.get(sor.loading_task_id) + ) save_requests.append(sr_dict) return save_requests diff --git a/swh/web/tests/api/views/test_origin_save.py b/swh/web/tests/api/views/test_origin_save.py --- a/swh/web/tests/api/views/test_origin_save.py +++ b/swh/web/tests/api/views/test_origin_save.py @@ -55,14 +55,15 @@ api_client, mocker, origin_url, - scheduler_task_status, expected_request_status, + scheduler_task_status=None, + scheduler_task_run_status=None, expected_task_status=None, visit_date=None, ): mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") - if not scheduler_task_status: + if scheduler_task_status is None: mock_scheduler.get_tasks.return_value = [] else: mock_scheduler.get_tasks.return_value = [ @@ -76,6 +77,22 @@ } ] + if scheduler_task_run_status is None: + mock_scheduler.get_task_runs.return_value = [] + else: + mock_scheduler.get_task_runs.return_value = [ + { + "backend_id": "f00c712c-e820-41ce-a07c-9bf8df914205", + "ended": datetime.now(tz=timezone.utc) + timedelta(minutes=5), + "id": 1, + "metadata": {}, + "scheduled": datetime.now(tz=timezone.utc), + "started": None, + "status": scheduler_task_run_status, + "task": 1, + } + ] + mock_scheduler.create_tasks.return_value = [ { "priority": "high", @@ -111,6 +128,7 @@ expected_request_status, expected_task_status, scheduler_task_status="next_run_not_scheduled", + scheduler_task_run_status=None, visit_date=None, ): mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") @@ -125,6 +143,22 @@ } ] + if scheduler_task_run_status is None: + mock_scheduler.get_task_runs.return_value = [] + else: + mock_scheduler.get_task_runs.return_value = [ + { + "backend_id": "f00c712c-e820-41ce-a07c-9bf8df914205", + "ended": datetime.now(tz=timezone.utc) + timedelta(minutes=5), + "id": 1, + "metadata": {}, + "scheduled": datetime.now(tz=timezone.utc), + "started": None, + "status": scheduler_task_run_status, + "task": 1, + } + ] + url = reverse( "api-1-save-origin", url_args={"visit_type": "git", "origin_url": origin_url} ) @@ -150,10 +184,14 @@ def test_save_request_rejected(api_client, mocker): origin_url = "https://github.com/user/illegal_repo" check_created_save_request_status( - api_client, mocker, origin_url, None, SAVE_REQUEST_REJECTED + api_client, mocker, origin_url, expected_request_status=SAVE_REQUEST_REJECTED, ) check_save_request_status( - api_client, mocker, origin_url, SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_CREATED + api_client, + mocker, + origin_url, + expected_request_status=SAVE_REQUEST_REJECTED, + expected_task_status=SAVE_TASK_NOT_CREATED, ) @@ -163,12 +201,15 @@ api_client, mocker, origin_url, - None, - SAVE_REQUEST_PENDING, - SAVE_TASK_NOT_CREATED, + expected_request_status=SAVE_REQUEST_PENDING, + expected_task_status=SAVE_TASK_NOT_CREATED, ) check_save_request_status( - api_client, mocker, origin_url, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_CREATED + api_client, + mocker, + origin_url, + expected_request_status=SAVE_REQUEST_PENDING, + expected_task_status=SAVE_TASK_NOT_CREATED, ) @@ -178,25 +219,26 @@ api_client, mocker, origin_url, - None, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_NOT_YET_SCHEDULED, ) check_save_request_status( api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SCHEDULED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_SCHEDULED, scheduler_task_status="next_run_scheduled", + scheduler_task_run_status="scheduled", ) check_save_request_status( api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SUCCEEDED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_SUCCEEDED, scheduler_task_status="completed", + scheduler_task_run_status="eventful", visit_date=None, ) visit_date = datetime.now(tz=timezone.utc) + timedelta(hours=1) @@ -204,9 +246,10 @@ api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SUCCEEDED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_SUCCEEDED, scheduler_task_status="completed", + scheduler_task_run_status="eventful", visit_date=visit_date, ) @@ -217,25 +260,26 @@ api_client, mocker, origin_url, - None, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_NOT_YET_SCHEDULED, ) check_save_request_status( api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SCHEDULED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_SCHEDULED, scheduler_task_status="next_run_scheduled", + scheduler_task_run_status="scheduled", ) check_save_request_status( api_client, mocker, origin_url, - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_FAILED, + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_FAILED, scheduler_task_status="disabled", + scheduler_task_run_status="failed", ) @@ -252,9 +296,9 @@ api_client, mocker, origin_url, - "next_run_not_scheduled", - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, + scheduler_task_status="next_run_not_scheduled", + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_NOT_YET_SCHEDULED, ) sors = list( @@ -266,9 +310,10 @@ api_client, mocker, origin_url, - "next_run_scheduled", - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_SCHEDULED, + scheduler_task_status="next_run_scheduled", + scheduler_task_run_status="scheduled", + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_SCHEDULED, ) sors = list( SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) @@ -280,9 +325,9 @@ api_client, mocker, origin_url, - "completed", - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, + scheduler_task_status="completed", + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_NOT_YET_SCHEDULED, visit_date=visit_date, ) sors = list( @@ -295,9 +340,9 @@ api_client, mocker, origin_url, - "disabled", - SAVE_REQUEST_ACCEPTED, - SAVE_TASK_NOT_YET_SCHEDULED, + scheduler_task_status="disabled", + expected_request_status=SAVE_REQUEST_ACCEPTED, + expected_task_status=SAVE_TASK_NOT_YET_SCHEDULED, ) sors = list( SaveOriginRequest.objects.filter(visit_type="git", origin_url=origin_url) diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py --- a/swh/web/tests/common/test_origin_save.py +++ b/swh/web/tests/common/test_origin_save.py @@ -61,24 +61,22 @@ _get_save_origin_task_info_test(mocker, es_available=False) -def _mock_scheduler(mocker, task_status="completed", task_archived=False): +def _mock_scheduler( + mocker, task_status="completed", task_run_status="eventful", task_archived=False +): mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler") - task = ( - { - "arguments": {"args": [], "kwargs": {"repo_url": _origin_url},}, - "current_interval": timedelta(days=64), - "id": _task_id, - "next_run": datetime.now(tz=timezone.utc) + timedelta(days=64), - "policy": "oneshot", - "priority": "high", - "retries_left": 0, - "status": task_status, - "type": "load-git", - } - if not task_archived - else None - ) - mock_scheduler.get_tasks.return_value = [dict(task) if task else None] + task = { + "arguments": {"args": [], "kwargs": {"repo_url": _origin_url},}, + "current_interval": timedelta(days=64), + "id": _task_id, + "next_run": datetime.now(tz=timezone.utc) + timedelta(days=64), + "policy": "oneshot", + "priority": "high", + "retries_left": 0, + "status": task_status, + "type": "load-git", + } + mock_scheduler.get_tasks.return_value = [dict(task) if not task_archived else None] task_run = { "backend_id": "f00c712c-e820-41ce-a07c-9bf8df914205", @@ -87,10 +85,12 @@ "metadata": {}, "scheduled": datetime.now(tz=timezone.utc), "started": None, - "status": task_status, + "status": task_run_status, "task": _task_id, } - mock_scheduler.get_task_runs.return_value = [dict(task_run)] + mock_scheduler.get_task_runs.return_value = [ + dict(task_run) if not task_archived else None + ] return task, task_run @@ -219,7 +219,7 @@ sor.request_date = datetime.now(tz=timezone.utc) - timedelta(days=31) sor.save() - _mock_scheduler(mocker, task_status="disabled") + _mock_scheduler(mocker, task_status="disabled", task_run_status="failed") sors = get_save_origin_requests(_visit_type, _origin_url) @@ -242,7 +242,9 @@ ) # mock scheduler and archives - _mock_scheduler(mocker, task_status="next_run_scheduled") + _mock_scheduler( + mocker, task_status="next_run_scheduled", task_run_status="scheduled" + ) mock_archive = mocker.patch("swh.web.common.origin_save.archive") mock_archive.lookup_origin.return_value = {"url": _origin_url} mock_get_origin_visits = mocker.patch(