diff --git a/OAq b/OAq new file mode 100644 --- /dev/null +++ b/OAq @@ -0,0 +1,67 @@ +diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js +index ea3c22cc..d8bc6e08 100644 +--- a/swh/web/assets/src/bundles/save/index.js ++++ b/swh/web/assets/src/bundles/save/index.js +@@ -113,8 +113,9 @@ export function initOriginSave() { + name: 'info', + render: (data, type, row) => { + if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') { +- return '`; ++ return ``; + } else { + return ''; + } +@@ -138,8 +139,9 @@ export function initOriginSave() { + window.location.hash = '#requests'; + }); +  +- $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => { ++ $('#swh-origin-save-request-help-tab').on('shown.bs.tab', () => { + removeUrlFragment(); ++ $('.swh-save-request-info').popover('dispose'); + }); +  + let saveRequestAcceptedAlert = htmlAlert( +@@ -320,7 +322,9 @@ export function displaySaveRequestInfo(event, saveRequestId) { + } + $('.swh-save-request-info').popover('dispose'); + $(event.target).popover({ +- 'title': 'Save request task information', ++ 'title': 'Save request task information ' + ++ ''`, + 'content': `
 +
 +  +@@ -331,6 +335,15 @@ export function displaySaveRequestInfo(event, saveRequestId) { + 'placement': 'left', + 'sanitizeFn': swh.webapp.filterXSS + }); ++ ++ $(event.target).on('shown.bs.popover', function() { ++ console.log($(this)); ++ const popoverId = $(this).attr('aria-describedby'); ++ $(`#${popoverId} .mdi-close`).click(() => { ++ $(this).popover('dispose'); ++ }); ++ }); ++ + $(event.target).popover('show'); + fetch(saveRequestTaskInfoUrl) + .then(response => response.json()) +diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html +index 3ad03305..ead6ced3 100644 +--- a/swh/web/templates/misc/origin-save.html ++++ b/swh/web/templates/misc/origin-save.html +@@ -59,7 +59,7 @@ See top-level LICENSE file for more information +
 +  +  +  diff --git a/cypress/fixtures/save-task-info.json b/cypress/fixtures/save-task-info.json new file mode 100644 --- /dev/null +++ b/cypress/fixtures/save-task-info.json @@ -0,0 +1,16 @@ +{ + "scheduled": "2020-06-24T11:48:12.561643+00:00", + "started": "2020-06-24T12:59:11.103188+00:00", + "ended": "2020-06-24T12:59:12.065313+00:00", + "status": "eventful", + "type": "load-git", + "arguments": { + "args": [], + "kwargs": { + "url": "https://gitlab.inria.fr/solverstack/maphys/maphys/" + } + }, + "duration": "1.0600971020758152", + "message": "[2020-06-24 12:59:12,063: INFO/ForkPoolWorker-161] Task swh.loader.git.tasks.UpdateGitRepository[4ff8b555-9535-4e75-b8ec-8e76165e14ec] succeeded in 1.0600971020758152s: {'status': 'eventful'}", + "name": "swh.loader.git.tasks.UpdateGitRepository" +} \ No newline at end of file diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -159,34 +159,76 @@ }); it('should display origin save info in the requests table', function() { - cy.fixture('origin-save').then(originSaveJSON => { - cy.route('GET', '/save/requests/list/**', originSaveJSON); - cy.get('#swh-origin-save-requests-list-tab').click(); - cy.get('tbody tr').then(rows => { - let i = 0; - for (let row of rows) { - const cells = row.cells; - const requestDateStr = new Date(originSaveJSON.data[i].save_request_date).toLocaleString(); - const saveStatus = originSaveJSON.data[i].save_task_status; - assert.equal($(cells[0]).text(), requestDateStr); - assert.equal($(cells[1]).text(), originSaveJSON.data[i].visit_type); - let html = ''; - if (saveStatus === 'succeed') { - let browseOriginUrl = `${this.Urls.browse_origin()}?origin_url=${originSaveJSON.data[i].origin_url}`; - browseOriginUrl += `&timestamp=${originSaveJSON.data[i].visit_date}`; - html += `${originSaveJSON.data[i].origin_url}`; - } else { - html += originSaveJSON.data[i].origin_url; - } - html += ` `; - html += ''; - assert.equal($(cells[2]).html(), html); - assert.equal($(cells[3]).text(), originSaveJSON.data[i].save_request_status); - assert.equal($(cells[4]).text(), saveStatus); - ++i; + cy.fixture('origin-save').as('originSaveJSON'); + cy.route('GET', '/save/requests/list/**', '@originSaveJSON'); + cy.get('#swh-origin-save-requests-list-tab').click(); + cy.get('tbody tr').then(rows => { + let i = 0; + for (let row of rows) { + const cells = row.cells; + const requestDateStr = new Date(this.originSaveJSON.data[i].save_request_date).toLocaleString(); + const saveStatus = this.originSaveJSON.data[i].save_task_status; + assert.equal($(cells[0]).text(), requestDateStr); + assert.equal($(cells[1]).text(), this.originSaveJSON.data[i].visit_type); + let html = ''; + if (saveStatus === 'succeed') { + let browseOriginUrl = `${this.Urls.browse_origin()}?origin_url=${this.originSaveJSON.data[i].origin_url}`; + browseOriginUrl += `&timestamp=${this.originSaveJSON.data[i].visit_date}`; + html += `${this.originSaveJSON.data[i].origin_url}`; + } else { + html += this.originSaveJSON.data[i].origin_url; } - }); + html += ` `; + html += ''; + assert.equal($(cells[2]).html(), html); + assert.equal($(cells[3]).text(), this.originSaveJSON.data[i].save_request_status); + assert.equal($(cells[4]).text(), saveStatus); + ++i; + } }); }); + it('should display/close task info popover when clicking on the info button', function() { + cy.fixture('origin-save').as('originSaveJSON'); + cy.fixture('save-task-info').as('saveTaskInfoJSON'); + cy.route('GET', '/save/requests/list/**', '@originSaveJSON'); + cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON'); + + cy.get('#swh-origin-save-requests-list-tab').click(); + cy.get('.swh-save-request-info') + .eq(0) + .click(); + + cy.get('.swh-save-request-info-popover') + .should('be.visible'); + + cy.get('.swh-save-request-info') + .eq(0) + .click(); + + cy.get('.swh-save-request-info-popover') + .should('not.be.visible'); + }); + + it('should hide task info popover when clicking on the close button', function() { + cy.fixture('origin-save').as('originSaveJSON'); + cy.fixture('save-task-info').as('saveTaskInfoJSON'); + cy.route('GET', '/save/requests/list/**', '@originSaveJSON'); + cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON'); + + cy.get('#swh-origin-save-requests-list-tab').click(); + cy.get('.swh-save-request-info') + .eq(0) + .click(); + + cy.get('.swh-save-request-info-popover') + .should('be.visible'); + + cy.get('.swh-save-request-info-close') + .click(); + + cy.get('.swh-save-request-info-popover') + .should('not.be.visible'); + }); + }); diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -23,7 +23,6 @@ from swh.web.common.origin_save import ( create_save_origin_request, - get_save_origin_task_info, SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED, ) @@ -214,16 +213,3 @@ entry.delete() status_code = 200 return HttpResponse(status=status_code) - - -@admin_route( - r"origin/save/task/info/(?P.+)/", - view_name="admin-origin-save-task-info", -) -@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) -def _save_origin_task_info(request, save_request_id): - request_info = get_save_origin_task_info(save_request_id) - for date_field in ("scheduled", "started", "ended"): - if date_field in request_info and request_info[date_field] is not None: - request_info[date_field] = request_info[date_field].isoformat() - return HttpResponse(json.dumps(request_info), content_type="application/json") diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js --- a/swh/web/assets/src/bundles/admin/origin-save.js +++ b/swh/web/assets/src/bundles/admin/origin-save.js @@ -150,7 +150,7 @@ render: (data, type, row) => { if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') { return '`; + `onclick="swh.save.displaySaveRequestInfo(event, ${row.id})">`; } else { return ''; } @@ -359,93 +359,3 @@ export function removeRejectedOriginSaveRequest() { removeOriginSaveRequest(rejectedSaveRequestsTable); } - -export function displaySaveRequestInfo(event, saveRequestId) { - event.stopPropagation(); - const saveRequestTaskInfoUrl = Urls.admin_origin_save_task_info(saveRequestId); - $('.swh-save-request-info').popover('dispose'); - $(event.target).popover({ - 'title': 'Save request task information', - 'content': `
-
- -

Fetching task information ...

-
-
`, - 'html': true, - 'placement': 'left', - 'sanitizeFn': swh.webapp.filterXSS - }); - $(event.target).popover('show'); - fetch(saveRequestTaskInfoUrl) - .then(response => response.json()) - .then(saveRequestTaskInfo => { - let content; - if ($.isEmptyObject(saveRequestTaskInfo)) { - content = 'Not available'; - } else { - let saveRequestInfo = []; - saveRequestInfo.push({ - key: 'Task type', - value: saveRequestTaskInfo.type - }); - if (saveRequestTaskInfo.hasOwnProperty('task_name')) { - saveRequestInfo.push({ - key: 'Task name', - value: saveRequestTaskInfo.name - }); - } - saveRequestInfo.push({ - key: 'Task arguments', - value: JSON.stringify(saveRequestTaskInfo.arguments, null, 2) - }); - saveRequestInfo.push({ - key: 'Task id', - value: saveRequestTaskInfo.id - }); - saveRequestInfo.push({ - key: 'Task backend id', - value: saveRequestTaskInfo.backend_id - }); - saveRequestInfo.push({ - key: 'Task scheduling date', - value: new Date(saveRequestTaskInfo.scheduled).toLocaleString() - }); - saveRequestInfo.push({ - key: 'Task termination date', - value: new Date(saveRequestTaskInfo.ended).toLocaleString() - }); - if (saveRequestTaskInfo.hasOwnProperty('duration')) { - saveRequestInfo.push({ - key: 'Task duration', - value: saveRequestTaskInfo.duration + ' s' - }); - } - if (saveRequestTaskInfo.hasOwnProperty('worker')) { - saveRequestInfo.push({ - key: 'Task executor', - value: saveRequestTaskInfo.worker - }); - } - if (saveRequestTaskInfo.hasOwnProperty('message')) { - saveRequestInfo.push({ - key: 'Task log', - value: saveRequestTaskInfo.message - }); - } - content = ''; - for (let info of saveRequestInfo) { - content += - ` - - - `; - } - content += '
${info.key} - 
${info.value}
-
'; - } - $('.swh-popover').html(content); - $(event.target).popover('update'); - }); -} diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js --- a/swh/web/assets/src/bundles/save/index.js +++ b/swh/web/assets/src/bundles/save/index.js @@ -108,6 +108,18 @@ { data: 'save_task_status', name: 'loading_task_status' + }, + { + name: 'info', + render: (data, type, row) => { + if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') { + return ``; + } else { + return ''; + } + } } ], scrollY: '50vh', @@ -127,8 +139,9 @@ window.location.hash = '#requests'; }); - $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => { + $('#swh-origin-save-request-help-tab').on('shown.bs.tab', () => { removeUrlFragment(); + $('.swh-save-request-info').popover('dispose'); }); let saveRequestAcceptedAlert = htmlAlert( @@ -298,3 +311,123 @@ }); }); } + +export function displaySaveRequestInfo(event, saveRequestId) { + event.stopPropagation(); + const saveRequestTaskInfoUrl = Urls.origin_save_task_info(saveRequestId); + // close popover when clicking again on the info icon + if ($(event.target).data('bs.popover')) { + $(event.target).popover('dispose'); + return; + } + $('.swh-save-request-info').popover('dispose'); + $(event.target).popover({ + animation: false, + boundary: 'viewport', + container: 'body', + title: 'Save request task information ' + + '`, + content: `
+
+ +

Fetching task information ...

+
+
`, + html: true, + placement: 'left', + sanitizeFn: swh.webapp.filterXSS + }); + + $(event.target).on('shown.bs.popover', function() { + const popoverId = $(this).attr('aria-describedby'); + $(`#${popoverId} .mdi-close`).click(() => { + $(this).popover('dispose'); + }); + }); + + $(event.target).popover('show'); + fetch(saveRequestTaskInfoUrl) + .then(response => response.json()) + .then(saveRequestTaskInfo => { + let content; + if ($.isEmptyObject(saveRequestTaskInfo)) { + content = 'Not available'; + } else { + let saveRequestInfo = []; + if (saveRequestTaskInfo.type) { + saveRequestInfo.push({ + key: 'Task type', + value: saveRequestTaskInfo.type + }); + } + if (saveRequestTaskInfo.arguments) { + saveRequestInfo.push({ + key: 'Task arguments', + value: JSON.stringify(saveRequestTaskInfo.arguments, null, 2) + }); + } + if (saveRequestTaskInfo.id) { + saveRequestInfo.push({ + key: 'Task id', + value: saveRequestTaskInfo.id + }); + } + if (saveRequestTaskInfo.backend_id) { + saveRequestInfo.push({ + key: 'Task backend id', + value: saveRequestTaskInfo.backend_id + }); + } + if (saveRequestTaskInfo.scheduled) { + saveRequestInfo.push({ + key: 'Task scheduling date', + value: new Date(saveRequestTaskInfo.scheduled).toLocaleString() + }); + } + if (saveRequestTaskInfo.started) { + saveRequestInfo.push({ + key: 'Task start date', + value: new Date(saveRequestTaskInfo.started).toLocaleString() + }); + } + if (saveRequestTaskInfo.ended) { + saveRequestInfo.push({ + key: 'Task termination date', + value: new Date(saveRequestTaskInfo.ended).toLocaleString() + }); + } + if (saveRequestTaskInfo.duration) { + saveRequestInfo.push({ + key: 'Task duration', + value: saveRequestTaskInfo.duration + ' seconds' + }); + } + if (saveRequestTaskInfo.worker) { + saveRequestInfo.push({ + key: 'Task executor', + value: saveRequestTaskInfo.worker + }); + } + if (saveRequestTaskInfo.message) { + saveRequestInfo.push({ + key: 'Task log', + value: saveRequestTaskInfo.message + }); + } + content = ''; + for (let info of saveRequestInfo) { + content += + ` + + + `; + } + content += '
${info.key} + 
${info.value}
+
'; + } + $('.swh-popover').html(content); + $(event.target).popover('update'); + }); +} diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -8,6 +8,7 @@ from itertools import product import json import logging +from typing import Any, Dict from django.core.exceptions import ObjectDoesNotExist from django.core.exceptions import ValidationError @@ -424,7 +425,9 @@ return get_save_origin_requests_from_queryset(sors) -def get_save_origin_task_info(save_request_id): +def get_save_origin_task_info( + save_request_id: int, full_info: bool = True +) -> Dict[str, Any]: """ Get detailed information about an accepted save origin request and its associated loading task. @@ -433,10 +436,11 @@ from the scheduler database, returns an empty dictionary. Args: - save_request_id (int): identifier of a save origin request + save_request_id: identifier of a save origin request + full_info: whether to return detailed info for staff users Returns: - dict: A dictionary with the following keys: + A dictionary with the following keys: - **type**: loading task type - **arguments**: loading task arguments @@ -474,7 +478,6 @@ task_run["id"] = task_run["task"] del task_run["task"] del task_run["metadata"] - del task_run["started"] es_workers_index_url = config.get_config()["es_workers_index_url"] if not es_workers_index_url: @@ -487,8 +490,8 @@ else: min_ts = save_request.request_date max_ts = min_ts + timedelta(days=30) - min_ts = int(min_ts.timestamp()) * 1000 - max_ts = int(max_ts.timestamp()) * 1000 + min_ts_unix = int(min_ts.timestamp()) * 1000 + max_ts_unix = int(max_ts.timestamp()) * 1000 save_task_status = _save_task_status[task["status"]] priority = "3" if save_task_status == SAVE_TASK_FAILED else "6" @@ -501,8 +504,8 @@ { "range": { "@timestamp": { - "gte": min_ts, - "lte": max_ts, + "gte": min_ts_unix, + "lte": max_ts_unix, "format": "epoch_millis", } } @@ -537,6 +540,21 @@ logger.warning("Request to Elasticsearch failed\n%s", exc) sentry_sdk.capture_exception(exc) + if not full_info: + for field in ("id", "backend_id", "worker"): + # remove some staff only fields + task_run.pop(field, None) + if "message" in task_run and "Loading failure" in task_run["message"]: + # hide traceback for non staff users, only display exception + message_lines = task_run["message"].split("\n") + message = "" + for line in message_lines: + if line.startswith("Traceback"): + break + message += f"{line}\n" + message += message_lines[-1] + task_run["message"] = message + return task_run diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py --- a/swh/web/misc/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -19,6 +19,7 @@ create_save_origin_request, get_savable_visit_types, get_save_origin_requests_from_queryset, + get_save_origin_task_info, ) from swh.web.common.utils import EnforceCSRFAuthentication @@ -101,6 +102,16 @@ return HttpResponse(table_data_json, content_type="application/json") +def _save_origin_task_info(request, save_request_id): + request_info = get_save_origin_task_info( + save_request_id, full_info=request.user.is_staff + ) + for date_field in ("scheduled", "started", "ended"): + if date_field in request_info and request_info[date_field] is not None: + request_info[date_field] = request_info[date_field].isoformat() + return HttpResponse(json.dumps(request_info), content_type="application/json") + + urlpatterns = [ url(r"^save/$", _origin_save_view, name="origin-save"), url( @@ -114,4 +125,9 @@ _origin_save_requests_list, name="origin-save-requests-list", ), + url( + r"^save/task/info/(?P.+)/", + _save_origin_task_info, + name="origin-save-task-info", + ), ] diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -59,7 +59,7 @@
@@ -109,6 +109,7 @@ Url Request Status + Info diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py --- a/swh/web/tests/common/test_origin_save.py +++ b/swh/web/tests/common/test_origin_save.py @@ -46,10 +46,15 @@ @pytest.mark.django_db -def test_get_save_origin_task_info_with_es(mocker): +def test_get_save_origin_task_full_info_with_es(mocker): _get_save_origin_task_info_test(mocker, es_available=True) +@pytest.mark.django_db +def test_get_save_origin_task_info_with_es(mocker): + _get_save_origin_task_info_test(mocker, es_available=True, full_info=False) + + @pytest.mark.django_db def test_get_save_origin_task_info_without_es(mocker): _get_save_origin_task_info_test(mocker, es_available=False) @@ -72,7 +77,7 @@ if not task_archived else None ) - mock_scheduler.get_tasks.return_value = [task] + mock_scheduler.get_tasks.return_value = [dict(task) if task else None] task_run = { "backend_id": "f00c712c-e820-41ce-a07c-9bf8df914205", @@ -84,11 +89,13 @@ "status": task_status, "task": _task_id, } - mock_scheduler.get_task_runs.return_value = [task_run] + mock_scheduler.get_task_runs.return_value = [dict(task_run)] return task, task_run -def _get_save_origin_task_info_test(mocker, task_archived=False, es_available=True): +def _get_save_origin_task_info_test( + mocker, task_archived=False, es_available=True, full_info=True +): swh_web_config = get_config() if es_available: @@ -111,7 +118,7 @@ task_exec_data = es_response["hits"]["hits"][-1]["_source"] - sor_task_info = get_save_origin_task_info(sor.id) + sor_task_info = get_save_origin_task_info(sor.id, full_info=full_info) expected_result = ( { @@ -120,6 +127,7 @@ "id": task["id"], "backend_id": task_run["backend_id"], "scheduled": task_run["scheduled"], + "started": task_run["started"], "ended": task_run["ended"], "status": task_run["status"], } @@ -136,6 +144,20 @@ } ) + if not full_info: + expected_result.pop("id", None) + expected_result.pop("backend_id", None) + expected_result.pop("worker", None) + if "message" in expected_result: + message = "" + message_lines = expected_result["message"].split("\n") + for line in message_lines: + if line.startswith("Traceback"): + break + message += f"{line}\n" + message += message_lines[-1] + expected_result["message"] = message + assert sor_task_info == expected_result