diff --git a/assets/src/bundles/admin/forge-add.js b/assets/src/bundles/admin/forge-add.js new file mode 100644 --- /dev/null +++ b/assets/src/bundles/admin/forge-add.js @@ -0,0 +1,395 @@ +/** + * Copyright (C) 2021 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import {handleFetchError, csrfPost, htmlAlert} from 'utils/functions'; +import {swhSpinnerSrc} from 'utils/constants'; + +let authorizedForgeTable; +let unauthorizedForgeTable; +let pendingAddForgeRequestsTable; +let acceptedAddForgeRequestsTable; +let rejectedAddForgeRequestsTable; + +function enableRowSelection(tableSel) { + $(`${tableSel} tbody`).on('click', 'tr', function() { + if ($(this).hasClass('selected')) { + $(this).removeClass('selected'); + $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', true); + } else { + $(`${tableSel} tr.selected`).removeClass('selected'); + $(this).addClass('selected'); + $(tableSel).closest('.tab-pane').find('.swh-action-need-selection').prop('disabled', false); + } + }); +} + +export function initForgeAddAdmin() { + $(document).ready(() => { + + $.fn.dataTable.ext.errMode = 'throw'; + + authorizedForgeTable = $('#swh-authorized-forge-urls').DataTable({ + serverSide: true, + ajax: Urls.admin_forge_add_authorized_urls_list(), + columns: [{data: 'url', name: 'url'}], + scrollY: '50vh', + scrollCollapse: true, + info: false + }); + enableRowSelection('#swh-authorized-forge-urls'); + swh.webapp.addJumpToPagePopoverToDataTable(authorizedForgeTable); + + unauthorizedForgeTable = $('#swh-unauthorized-forge-urls').DataTable({ + serverSide: true, + ajax: Urls.admin_forge_add_unauthorized_urls_list(), + columns: [{data: 'url', name: 'url'}], + scrollY: '50vh', + scrollCollapse: true, + info: false + }); + enableRowSelection('#swh-unauthorized-forge-urls'); + swh.webapp.addJumpToPagePopoverToDataTable(unauthorizedForgeTable); + + const columnsData = [ + { + data: 'id', + name: 'id', + visible: false, + searchable: false + }, + { + data: 'request_date', + name: 'request_date', + render: (data, type, row) => { + if (type === 'display') { + const date = new Date(data); + return date.toLocaleString(); + } + return data; + } + }, + { + data: 'forge_type', + name: 'forge_type' + }, + { + data: 'forge_url', + name: 'forge_url' + } + ]; + + pendingAddForgeRequestsTable = $('#swh-forge-add-pending-requests').DataTable({ + serverSide: true, + processing: true, + language: { + processing: `` + }, + ajax: Urls.forge_add_requests_list('pending'), + searchDelay: 1000, + columns: columnsData, + scrollY: '50vh', + scrollCollapse: true, + order: [[0, 'desc']], + responsive: { + details: { + type: 'none' + } + } + }); + enableRowSelection('#swh-forge-add-pending-requests'); + swh.webapp.addJumpToPagePopoverToDataTable(pendingAddForgeRequestsTable); + + columnsData.push({ + name: 'info', + render: (data, type, row) => { + if (row.add_task_status === 'succeeded' || row.add_task_status === 'failed' || + row.note != null) { + return ``; + } else { + return ''; + } + } + }); + + rejectedAddForgeRequestsTable = $('#swh-forge-add-rejected-requests').DataTable({ + serverSide: true, + processing: true, + language: { + processing: `` + }, + ajax: Urls.forge_add_requests_list('rejected'), + searchDelay: 1000, + columns: columnsData, + scrollY: '50vh', + scrollCollapse: true, + order: [[0, 'desc']], + responsive: { + details: { + type: 'none' + } + } + }); + enableRowSelection('#swh-forge-add-rejected-requests'); + swh.webapp.addJumpToPagePopoverToDataTable(rejectedAddForgeRequestsTable); + + columnsData.splice(columnsData.length - 1, 0, { + data: 'add_task_status', + name: 'add_task_status' + }); + + acceptedAddForgeRequestsTable = $('#swh-forge-add-accepted-requests').DataTable({ + serverSide: true, + processing: true, + language: { + processing: `` + }, + ajax: Urls.forge_add_requests_list('accepted'), + searchDelay: 1000, + columns: columnsData, + scrollY: '50vh', + scrollCollapse: true, + order: [[0, 'desc']], + responsive: { + details: { + type: 'none' + } + } + }); + enableRowSelection('#swh-forge-add-accepted-requests'); + swh.webapp.addJumpToPagePopoverToDataTable(acceptedAddForgeRequestsTable); + + $('#swh-forge-add-requests-nav-item').on('shown.bs.tab', () => { + pendingAddForgeRequestsTable.draw(); + }); + + $('#swh-forge-add-url-filters-nav-item').on('shown.bs.tab', () => { + authorizedForgeTable.draw(); + }); + + $('#swh-authorized-forges-tab').on('shown.bs.tab', () => { + authorizedForgeTable.draw(); + }); + + $('#swh-unauthorized-forges-tab').on('shown.bs.tab', () => { + unauthorizedForgeTable.draw(); + }); + + $('#swh-forge-add-requests-pending-tab').on('shown.bs.tab', () => { + pendingAddForgeRequestsTable.draw(); + }); + + $('#swh-forge-add-requests-accepted-tab').on('shown.bs.tab', () => { + acceptedAddForgeRequestsTable.draw(); + }); + + $('#swh-forge-add-requests-rejected-tab').on('shown.bs.tab', () => { + rejectedAddForgeRequestsTable.draw(); + }); + + $('#swh-forge-add-requests-pending-tab').click(() => { + pendingAddForgeRequestsTable.ajax.reload(null, false); + }); + + $('#swh-forge-add-requests-accepted-tab').click(() => { + acceptedAddForgeRequestsTable.ajax.reload(null, false); + }); + + $('#swh-forge-add-requests-rejected-tab').click(() => { + rejectedAddForgeRequestsTable.ajax.reload(null, false); + }); + + $('body').on('click', e => { + if ($(e.target).parents('.popover').length > 0) { + e.stopPropagation(); + } else if ($(e.target).parents('.swh-add-forge-request-info').length === 0) { + $('.swh-add-forge-request-info').popover('dispose'); + } + }); + + }); +} + +export async function addAuthorizedForgeUrl() { + const forgeUrl = $('#swh-authorized-url-prefix').val(); + const addForgeUrl = Urls.admin_forge_add_add_authorized_url(forgeUrl); + try { + const response = await csrfPost(addForgeUrl); + handleFetchError(response); + authorizedForgeTable.row.add({'url': forgeUrl}).draw(); + $('.swh-add-authorized-forge-status').html( + htmlAlert('success', 'The forge url has been successfully added in the authorized list.', true) + ); + } catch (_) { + $('.swh-add-authorized-forge-status').html( + htmlAlert('warning', 'The provided forge url prefix is already registered in the authorized list.', true) + ); + } +} + +export async function removeAuthorizedForgeUrl() { + const forgeUrl = $('#swh-authorized-forge-urls tr.selected').text(); + if (forgeUrl) { + const removeForgeUrl = Urls.admin_forge_add_remove_authorized_url(forgeUrl); + try { + const response = await csrfPost(removeForgeUrl); + handleFetchError(response); + authorizedForgeTable.row('.selected').remove().draw(); + } catch (_) {} + } +} + +export async function addUnauthorizedForgeUrl() { + const forgeUrl = $('#swh-unauthorized-url-prefix').val(); + const addForgeUrl = Urls.admin_forge_add_add_unauthorized_url(forgeUrl); + try { + const response = await csrfPost(addForgeUrl); + handleFetchError(response); + unauthorizedForgeTable.row.add({'url': forgeUrl}).draw(); + $('.swh-add-unauthorized-forge-status').html( + htmlAlert('success', 'The forge url prefix has been successfully added in the unauthorized list.', true) + ); + } catch (_) { + $('.swh-add-unauthorized-forge-status').html( + htmlAlert('warning', 'The provided forge url prefix is already registered in the unauthorized list.', true) + ); + } +} + +export async function removeUnauthorizedForgeUrl() { + const forgeUrl = $('#swh-unauthorized-forge-urls tr.selected').text(); + if (forgeUrl) { + const removeForgeUrl = Urls.admin_forge_add_remove_unauthorized_url(forgeUrl); + try { + const response = await csrfPost(removeForgeUrl); + handleFetchError(response); + unauthorizedForgeTable.row('.selected').remove().draw(); + } catch (_) {}; + } +} + +export function acceptForgeAddRequest() { + const selectedRow = pendingAddForgeRequestsTable.row('.selected'); + if (selectedRow.length) { + const acceptForgeAddRequestCallback = async() => { + const rowData = selectedRow.data(); + const acceptAddRequestUrl = Urls.admin_forge_add_request_accept(rowData['visit_type'], rowData['forge_url']); + await csrfPost(acceptAddRequestUrl); + pendingAddForgeRequestsTable.ajax.reload(null, false); + }; + + swh.webapp.showModalConfirm( + 'Accept forge add request ?', + 'Are you sure to accept this forge add request ?', + acceptForgeAddRequestCallback); + } +} + +const rejectModalHtml = ` +
+
+ +
+ +
+
+
+ +
+ +
+`; + +export function rejectForgeAddRequest() { + const selectedRow = pendingAddForgeRequestsTable.row('.selected'); + const rowData = selectedRow.data(); + if (selectedRow.length) { + const rejectForgeAddRequestCallback = async() => { + $('#swh-web-modal-html').modal('hide'); + const rejectAddRequestUrl = Urls.admin_forge_add_request_reject( + rowData['visit_type'], rowData['forge_url']); + await csrfPost(rejectAddRequestUrl, {}, + JSON.stringify({note: $('#swh-rejection-text').val()})); + pendingAddForgeRequestsTable.ajax.reload(null, false); + }; + + let currentRejectionReason = 'custom'; + const rejectionTexts = {}; + swh.webapp.showModalHtml('Reject forge add request ?', rejectModalHtml); + $('#swh-rejection-reason').on('change', (event) => { + // backup current textarea value + rejectionTexts[currentRejectionReason] = $('#swh-rejection-text').val(); + currentRejectionReason = event.target.value; + let newRejectionText = ''; + if (rejectionTexts.hasOwnProperty(currentRejectionReason)) { + // restore previous textarea value + newRejectionText = rejectionTexts[currentRejectionReason]; + } else { + // fill textarea with default text according to rejection type + if (currentRejectionReason === 'invalid-forge') { + newRejectionText = `The forge with URL ${rowData['forge_url']} is not ` + + `a link to a ${rowData['visit_type']} repository.`; + } else if (currentRejectionReason === 'invalid-forge-type') { + newRejectionText = `The forge with URL ${rowData['forge_url']} is not ` + + `of type ${rowData['visit_type']}.`; + } else if (currentRejectionReason === 'forge-not-found') { + newRejectionText = `The forge with URL ${rowData['forge_url']} cannot be found.`; + } + } + $('#swh-rejection-text').val(newRejectionText); + }); + $('#swh-rejection-form').on('submit', (event) => { + event.preventDefault(); + event.stopPropagation(); + // ensure confirmation modal will be displayed above the html modal + $('#swh-web-modal-html').css('z-index', 4000); + swh.webapp.showModalConfirm( + 'Reject forge add request ?', + 'Are you sure to reject this forge add request ?', + rejectForgeAddRequestCallback); + }); + } +} + +function removeForgeAddRequest(requestTable) { + const selectedRow = requestTable.row('.selected'); + if (selectedRow.length) { + const requestId = selectedRow.data()['id']; + const removeForgeAddRequestCallback = async() => { + const removeAddRequestUrl = Urls.admin_forge_add_request_remove(requestId); + await csrfPost(removeAddRequestUrl); + requestTable.ajax.reload(null, false); + }; + + swh.webapp.showModalConfirm( + 'Remove forge add request ?', + 'Are you sure to remove this forge add request ?', + removeForgeAddRequestCallback); + } +} + +export function removePendingForgeAddRequest() { + removeForgeAddRequest(pendingAddForgeRequestsTable); +} + +export function removeAcceptedForgeAddRequest() { + removeForgeAddRequest(acceptedAddForgeRequestsTable); +} + +export function removeRejectedForgeAddRequest() { + removeForgeAddRequest(rejectedAddForgeRequestsTable); +} diff --git a/assets/src/bundles/admin/index.js b/assets/src/bundles/admin/index.js --- a/assets/src/bundles/admin/index.js +++ b/assets/src/bundles/admin/index.js @@ -1,5 +1,5 @@ /** - * Copyright (C) 2018 The Software Heritage developers + * Copyright (C) 2018-2021 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information @@ -7,3 +7,4 @@ export * from './deposit'; export * from './origin-save'; +export * from './forge-add'; diff --git a/assets/src/bundles/forge_add/index.js b/assets/src/bundles/forge_add/index.js new file mode 100644 --- /dev/null +++ b/assets/src/bundles/forge_add/index.js @@ -0,0 +1,427 @@ +/** + * Copyright (C) 2021 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +import {csrfPost, handleFetchError, htmlAlert, removeUrlFragment} from 'utils/functions'; +import {swhSpinnerSrc} from 'utils/constants'; + +let addForgeRequestsTable; + +async function forgeAddRequest( + forgeType, forgeUrl, acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the forge add request + const addForgeUrlRequestUrl = Urls.api_1_add_forge(forgeType, forgeUrl); + + // show computation spinner + $('.swh-processing-forge-add-request').css('display', 'block'); + const headers = {}; + const body = null; + + try { + const response = await csrfPost(addForgeUrlRequestUrl, headers, body); + handleFetchError(response); + const data = await response.json(); + // hide the computation spinner + $('.swh-processing-forge-add-request').css('display', 'none'); + if (data.add_forge_request_status === 'accepted') { + acceptedCallback(); + } else { + pendingCallback(); + } + } catch (response) { + $('.swh-processing-forge-add-request').css('display', 'none'); + const errorData = await response.json(); + errorCallback(response.status, errorData); + }; +} + +const userRequestsFilterCheckbox = ` +
+ + +
+`; + +export function initForgeAdd() { + + $(document).ready(() => { + + $.fn.dataTable.ext.errMode = 'none'; + + // set git as the default value as before + $('#swh-input-forge-type').val('gitlab'); + + addForgeRequestsTable = $('#swh-forge-add-requests') + .on('error.dt', (e, settings, techNote, message) => { + $('#swh-forge-add-request-list-error').text( + 'An error occurred while retrieving the add forge requests list' + ); + console.log(message); + }) + .DataTable({ + serverSide: true, + processing: true, + language: { + processing: `` + }, + ajax: { + url: Urls.forge_add_requests_list('all'), + data: (d) => { + if (swh.webapp.isUserLoggedIn() && $('#swh-add-forge-requests-user-filter').prop('checked')) { + d.user_requests_only = '1'; + } + } + }, + searchDelay: 1000, + dom: '<"row"<"col-sm-3"l><"col-sm-6 text-left user-requests-filter"><"col-sm-3"f>>' + + '<"row"<"col-sm-12"tr>>' + + '<"row"<"col-sm-5"i><"col-sm-7"p>>', + fnInitComplete: function() { + if (swh.webapp.isUserLoggedIn()) { + $('div.user-requests-filter').html(userRequestsFilterCheckbox); + $('#swh-add-forge-requests-user-filter').on('change', () => { + addForgeRequestsTable.draw(); + }); + } + }, + columns: [ + { + data: 'request_date', + name: 'request_date', + render: (data, type, row) => { + if (type === 'display') { + const date = new Date(data); + return date.toLocaleString(); + } + return data; + } + }, + { + data: 'forge_type', + name: 'forge_type' + }, + { + data: 'forge_url', + name: 'forge_url' + }, + { + data: 'request_status', + name: 'request_status' + }, + { + data: 'task_status', + name: 'task_status' + }, + { + name: 'info', + render: (data, type, row) => { + if (row.task_status === 'succeeded' || row.task_status === 'failed' || + row.note != null) { + return ``; + } else { + return ''; + } + } + } + ], + scrollY: '50vh', + scrollCollapse: true, + order: [[0, 'desc']], + responsive: { + details: { + type: 'none' + } + } + }); + + swh.webapp.addJumpToPagePopoverToDataTable(addForgeRequestsTable); + + $('#swh-forge-add-requests-list').on('shown.bs.tab', () => { + addForgeRequestsTable.draw(); + window.location.hash = '#forge-requests'; + }); + + $('#swh-forge-add-request-help').on('shown.bs.tab', () => { + removeUrlFragment(); + $('.swh-add-forge-request-info').popover('dispose'); + }); + + const addForgeRequestAcceptedAlert = htmlAlert( + 'success', + 'The "add forge now" request has been accepted and will be processed as soon as possible.', + true + ); + + const addForgeRequestPendingAlert = htmlAlert( + 'warning', + 'The "add forge now" request has been put in pending state and may be accepted for processing after manual review.', + true + ); + + const addForgeRequestRateLimitedAlert = htmlAlert( + 'danger', + 'The rate limit for "add forge now" requests has been reached. Please try again later.', + true + ); + + const addForgeRequestUnknownErrorAlert = htmlAlert( + 'danger', + 'An unexpected error happened when submitting the "add forge now request".', + true + ); + + $('#swh-add-forge-forge-form').submit(async event => { + event.preventDefault(); + event.stopPropagation(); + $('.alert').alert('close'); + if (event.target.checkValidity()) { + $(event.target).removeClass('was-validated'); + const forgeType = $('#swh-input-visit-type').val(); + const forgeUrl = $('#swh-input-forge-url').val(); + + forgeAddRequest( + forgeType, + forgeUrl, + () => $('#swh-forge-add-request-status').html(addForgeRequestAcceptedAlert), + () => $('#swh-forge-add-request-status').html(addForgeRequestPendingAlert), + (statusCode, errorData) => { + $('#swh-forge-add-request-status').css('color', 'red'); + if (statusCode === 403) { + const errorAlert = htmlAlert('danger', `Error: ${errorData['reason']}`); + $('#swh-forge-add-request-status').html(errorAlert); + } else if (statusCode === 429) { + $('#swh-forge-add-request-status').html(addForgeRequestRateLimitedAlert); + } else if (statusCode === 400) { + const errorAlert = htmlAlert('danger', errorData['reason']); + $('#swh-forge-add-request-status').html(errorAlert); + } else { + $('#swh-forge-add-request-status').html(addForgeRequestUnknownErrorAlert); + } + }); + } else { + $(event.target).addClass('was-validated'); + } + }); + + $('#swh-show-forge-add-requests-list').on('click', (event) => { + event.preventDefault(); + $('.nav-tabs a[href="#swh-forge-add-requests-list"]').tab('show'); + }); + + $('#swh-input-forge-url').on('input', function(event) { + const forgeUrl = $(this).val().trim(); + $(this).val(forgeUrl); + $('#swh-input-visit-type option').each(function() { + const val = $(this).val(); + if (val && forgeUrl.includes(val)) { + $(this).prop('selected', true); + } + }); + }); + + if (window.location.hash === '#forge-requests') { + $('.nav-tabs a[href="#swh-forge-add-requests-list"]').tab('show'); + } + + }); + +} + +export function validateAddForgeUrl(input) { + let forgeUrl = null; + let validUrl = true; + + try { + forgeUrl = new URL(input.value.trim()); + } catch (TypeError) { + validUrl = false; + } + + if (validUrl) { + const allowedProtocols = ['http:', 'https:']; + validUrl = ( + allowedProtocols.find(protocol => protocol === forgeUrl.protocol) !== undefined + ); + } + + if (validUrl) { + input.setCustomValidity(''); + } else { + input.setCustomValidity('The forge url is not valid or does not reference a code repository'); + } +} + +export function initTakeNewSnapshot() { + + const newSnapshotRequestAcceptedAlert = htmlAlert( + 'success', + 'The "take new snapshot" request has been accepted and will be processed as soon as possible.', + true + ); + + const newSnapshotRequestPendingAlert = htmlAlert( + 'warning', + 'The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.', + true + ); + + const newSnapshotRequestRateLimitAlert = htmlAlert( + 'danger', + 'The rate limit for "take new snapshot" requests has been reached. Please try again later.', + true + ); + + const newSnapshotRequestUnknownErrorAlert = htmlAlert( + 'danger', + 'An unexpected error happened when submitting the "add forge now request".', + true + ); + + $(document).ready(() => { + $('#swh-take-new-snapshot-form').submit(event => { + event.preventDefault(); + event.stopPropagation(); + + const forgeType = $('#swh-input-visit-type').val(); + const forgeUrl = $('#swh-input-forge-url').val(); + + forgeAddRequest(forgeType, forgeUrl, + () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), + () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), + (statusCode, errorData) => { + $('#swh-take-new-snapshot-request-status').css('color', 'red'); + if (statusCode === 403) { + const errorAlert = htmlAlert('danger', `Error: ${errorData['detail']}`, true); + $('#swh-take-new-snapshot-request-status').html(errorAlert); + } else if (statusCode === 429) { + $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRateLimitAlert); + } else { + $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestUnknownErrorAlert); + } + }); + }); + }); +} + +export function formatValuePerType(type, value) { + // Given some typed value, format and return accordingly formatted value + const mapFormatPerTypeFn = { + 'json': (v) => JSON.stringify(v, null, 2), + 'date': (v) => new Date(v).toLocaleString(), + 'raw': (v) => v, + 'duration': (v) => v + ' seconds' + }; + + return value === null ? null : mapFormatPerTypeFn[type](value); +} + +export async function displayForgeAddRequestInfo(event, addForgeRequestId) { + event.stopPropagation(); + const addForgeRequestTaskInfoUrl = Urls.forge_add_task_info(addForgeRequestId); + // close popover when clicking again on the info icon + if ($(event.target).data('bs.popover')) { + $(event.target).popover('dispose'); + return; + } + $('.swh-add-forge-request-info').popover('dispose'); + $(event.target).popover({ + animation: false, + boundary: 'viewport', + container: 'body', + title: 'Add request task information ' + + '`, + content: `
+
+ +

Fetching task information ...

+
+
`, + html: true, + placement: 'left', + sanitizeFn: swh.webapp.filterXSS + }); + + $(event.target).on('shown.bs.popover', function() { + const popoverId = $(this).attr('aria-describedby'); + $(`#${popoverId} .mdi-close`).click(() => { + $(this).popover('dispose'); + }); + }); + + $(event.target).popover('show'); + const response = await fetch(addForgeRequestTaskInfoUrl); + const addForgeRequestTaskInfo = await response.json(); + + let content; + if ($.isEmptyObject(addForgeRequestTaskInfo)) { + content = 'Not available'; + + } else if (addForgeRequestTaskInfo.note != null) { + content = addForgeRequestTaskInfo.note; + } else { + const addForgeRequestInfo = []; + const taskData = { + 'Type': ['raw', 'forge_type'], + 'Arguments': ['json', 'arguments'], + 'Id': ['raw', 'id'], + 'Scheduling date': ['date', 'scheduled'], + 'Start date': ['date', 'started'], + 'Completion date': ['date', 'ended'], + 'Duration': ['duration', 'duration'], + 'Runner': ['raw', 'worker'], + 'Log': ['raw', 'message'] + }; + for (const [title, [type, property]] of Object.entries(taskData)) { + if (addForgeRequestTaskInfo.hasOwnProperty(property)) { + addForgeRequestInfo.push({ + key: title, + value: formatValuePerType(type, addForgeRequestTaskInfo[property]) + }); + } + } + content = ''; + for (const info of addForgeRequestInfo) { + content += + ` + + + `; + } + content += '
'; + } + $('.swh-popover').html(content); + $(event.target).popover('update'); +} + +export function fillForgeAddRequestFormAndScroll(visitType, forgeUrl) { + $('#swh-input-forge-url').val(forgeUrl); + let forgeTypeFound = false; + $('#swh-input-visit-type option').each(function() { + const val = $(this).val(); + if (val && forgeUrl.includes(val)) { + $(this).prop('selected', true); + forgeTypeFound = true; + } + }); + if (!forgeTypeFound) { + $('#swh-input-visit-type option').each(function() { + const val = $(this).val(); + if (val === visitType) { + $(this).prop('selected', true); + } + }); + } + window.scrollTo(0, 0); +} diff --git a/swh/web/admin/forge_add.py b/swh/web/admin/forge_add.py new file mode 100644 --- /dev/null +++ b/swh/web/admin/forge_add.py @@ -0,0 +1,220 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import json + +from django.conf import settings +from django.contrib.admin.views.decorators import staff_member_required +from django.core.exceptions import ObjectDoesNotExist +from django.core.paginator import Paginator +from django.http import HttpResponse, JsonResponse +from django.shortcuts import render +from django.views.decorators.http import require_POST + +from swh.web.admin.adminurls import admin_route +from swh.web.common.models import ( + ADD_FORGE_REQUEST_PENDING, + ADD_FORGE_REQUEST_REJECTED, + AddAuthorizedForge, + AddForgeRequest, + AddUnauthorizedForge, +) + + +def create_add_forge_request(forge_type: str, forge_url: str) -> None: + """Schedule a listing forge task. + + """ + pass + + +@admin_route(r"forge/add/", view_name="admin-forge-add") +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add(request): + return render(request, "admin/forge-add.html") + + +def _datatables_forge_urls_response(request, urls_query_set): + search_value = request.GET["search[value]"] + if search_value: + urls_query_set = urls_query_set.filter(url__icontains=search_value) + + column_order = request.GET["order[0][column]"] + field_order = request.GET["columns[%s][name]" % column_order] + order_dir = request.GET["order[0][dir]"] + if order_dir == "desc": + field_order = "-" + field_order + + urls_query_set = urls_query_set.order_by(field_order) + + table_data = {} + table_data["draw"] = int(request.GET["draw"]) + table_data["recordsTotal"] = urls_query_set.count() + table_data["recordsFiltered"] = urls_query_set.count() + length = int(request.GET["length"]) + page = int(request.GET["start"]) / length + 1 + paginator = Paginator(urls_query_set, length) + urls_query_set = paginator.page(page).object_list + table_data["data"] = [{"url": u.url} for u in urls_query_set] + return JsonResponse(table_data) + + +@admin_route( + r"forge/add/authorized_urls/list/", + view_name="admin-forge-add-authorized-urls-list", +) +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_authorized_urls_list(request): + authorized_urls = AddAuthorizedForge.objects.all() + return _datatables_forge_urls_response(request, authorized_urls) + + +@admin_route( + r"forge/add/authorized_urls/add/(?P.+)/", + view_name="admin-forge-add-add-authorized-url", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_add_authorized_url(request, forge_url): + try: + AddAuthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + # add the new authorized url + AddAuthorizedForge.objects.create(url=forge_url) + # check if pending add requests with that url prefix exist + pending_add_requests = AddForgeRequest.objects.filter( + forge_url__startswith=forge_url, request_status=ADD_FORGE_REQUEST_PENDING + ) + # create forge add tasks for previously pending requests + for psr in pending_add_requests: + create_add_forge_request(psr.forge_type, psr.forge_url) + status_code = 200 + else: + status_code = 400 + return HttpResponse(status=status_code) + + +@admin_route( + r"forge/add/authorized_urls/remove/(?P.+)/", + view_name="admin-forge-add-remove-authorized-url", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_remove_authorized_url(request, forge_url): + try: + entry = AddAuthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + status_code = 404 + else: + entry.delete() + status_code = 200 + return HttpResponse(status=status_code) + + +@admin_route( + r"forge/add/unauthorized_urls/list/", + view_name="admin-forge-add-unauthorized-urls-list", +) +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_unauthorized_urls_list(request): + unauthorized_urls = AddUnauthorizedForge.objects.all() + return _datatables_forge_urls_response(request, unauthorized_urls) + + +@admin_route( + r"forge/add/unauthorized_urls/add/(?P.+)/", + view_name="admin-forge-add-add-unauthorized-url", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_add_unauthorized_url(request, forge_url): + try: + AddUnauthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + AddUnauthorizedForge.objects.create(url=forge_url) + # check if pending add requests with that url prefix exist + pending_add_requests = AddForgeRequest.objects.filter( + forge_url__startswith=forge_url, request_status=ADD_FORGE_REQUEST_PENDING + ) + # mark pending requests as rejected + for psr in pending_add_requests: + psr.status = ADD_FORGE_REQUEST_REJECTED + psr.add() + status_code = 200 + else: + status_code = 400 + return HttpResponse(status=status_code) + + +@admin_route( + r"forge/add/unauthorized_urls/remove/(?P.+)/", + view_name="admin-forge-add-remove-unauthorized-url", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_remove_unauthorized_url(request, forge_url): + try: + entry = AddUnauthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + status_code = 404 + else: + entry.delete() + status_code = 200 + return HttpResponse(status=status_code) + + +@admin_route( + r"forge/add/request/accept/(?P.+)/url/(?P.+)/", + view_name="admin-forge-add-request-accept", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_request_accept(request, forge_type, forge_url): + try: + AddAuthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + AddAuthorizedForge.objects.create(url=forge_url) + create_add_forge_request(forge_type, forge_url) + return HttpResponse(status=200) + + +@admin_route( + r"forge/add/request/reject/(?P.+)/url/(?P.+)/", + view_name="admin-forge-add-request-reject", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_request_reject(request, forge_type, forge_url): + try: + AddUnauthorizedForge.objects.get(url=forge_url) + except ObjectDoesNotExist: + AddUnauthorizedForge.objects.create(url=forge_url) + sor = AddForgeRequest.objects.get( + forge_type=forge_type, + forge_url=forge_url, + request_status=ADD_FORGE_REQUEST_PENDING, + ) + + sor.status = ADD_FORGE_REQUEST_REJECTED + sor.note = json.loads(request.body).get("note") + sor.add() + return HttpResponse(status=200) + + +@admin_route( + r"forge/add/request/remove/(?P.+)/", + view_name="admin-forge-add-request-remove", +) +@require_POST +@staff_member_required(view_func=None, login_url=settings.LOGIN_URL) +def _admin_forge_add_request_remove(request, sor_id): + try: + entry = AddForgeRequest.objects.get(id=sor_id) + except ObjectDoesNotExist: + status_code = 404 + else: + entry.delete() + status_code = 200 + return HttpResponse(status=status_code) diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py --- a/swh/web/admin/urls.py +++ b/swh/web/admin/urls.py @@ -1,4 +1,4 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,6 +9,7 @@ from swh.web.admin.adminurls import AdminUrls import swh.web.admin.deposit # noqa +import swh.web.admin.forge_add # noqa import swh.web.admin.origin_save # noqa diff --git a/swh/web/api/views/forge_add.py b/swh/web/api/views/forge_add.py new file mode 100644 --- /dev/null +++ b/swh/web/api/views/forge_add.py @@ -0,0 +1,467 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +from typing import Any, Dict, List, Optional + +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.core.validators import URLValidator +from django.db.models import QuerySet +from django.utils.html import escape + +from swh.scheduler.utils import create_oneshot_task_dict +from swh.web.api.apidoc import api_doc, format_docstring +from swh.web.api.apiurls import api_route +from swh.web.auth.utils import ( + API_ADD_FORGE_PERMISSION, + SWH_AMBASSADOR_PERMISSION, + privileged_user, +) +from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc +from swh.web.common.forge_add import get_forge_types +from swh.web.common.models import ( + ADD_FORGE_REQUEST_ACCEPTED, + ADD_FORGE_REQUEST_PENDING, + ADD_FORGE_REQUEST_REJECTED, + AddAuthorizedForge, + AddForgeRequest, + AddUnauthorizedForge, +) +from swh.web.common.typing import AddForgeRequestInfo +from swh.web.config import scheduler + +_validate_url = URLValidator(schemes=["http", "https"]) + + +def get_forge_add_authorized_urls() -> List[str]: + """Get the list of forge url prefixes authorized to be immediately loaded into the + archive (whitelist). + + Returns: + The list of authorized forge url prefixes + + """ + return [forge.url for forge in AddAuthorizedForge.objects.all()] + + +def get_forge_add_unauthorized_urls() -> List[str]: + """Get the list of forge url prefixes forbidden to be loaded into the archive + (blacklist). + + Returns: + the list of unauthorized forge url prefixes + + """ + return [forge.url for forge in AddUnauthorizedForge.objects.all()] + + +def can_add_forge(forge_url: str, bypass_pending_review: bool = False) -> str: + """str if a software forge can be added into the archive. + + Based on the forge url, the add request will be either: + + * immediately accepted if the url is whitelisted + * rejected if the url is blacklisted + * put in pending state for manual review otherwise + + Args: + forge_url (str): the software forge url to check + + Returns: + str: the forge add request request_status, either **accepted**, + **rejected** or **pending** + + """ + # forge url may be blacklisted + for url_prefix in get_forge_add_unauthorized_urls(): + if forge_url.startswith(url_prefix): + return ADD_FORGE_REQUEST_REJECTED + + # if the forge url is in the white list, it can be immediately added + for url_prefix in get_forge_add_authorized_urls(): + if forge_url.startswith(url_prefix): + return ADD_FORGE_REQUEST_ACCEPTED + + # otherwise, the forge url needs to be manually verified if the user + # that submitted it does not have special permission + if bypass_pending_review: + # mark the forge URL as trusted in that case + AddAuthorizedForge.objects.get_or_create(url=forge_url) + return ADD_FORGE_REQUEST_ACCEPTED + else: + return ADD_FORGE_REQUEST_PENDING + + +def _check_forge_type(forge_type: str, privileged_user: bool = False) -> None: + visit_type_tasks = get_forge_types(privileged_user) + if forge_type not in visit_type_tasks: + allowed_visit_types = ", ".join(visit_type_tasks) + raise BadInputExc( + f"Visit of type {forge_type} cannot be added! " + f"Allowed types are the following: {allowed_visit_types}" + ) + + +def _check_forge_url_valid(forge_url: str) -> None: + """Check the forge url is valid and raise if not.""" + try: + _validate_url(forge_url) + except ValidationError: + raise BadInputExc(f"The provided forge url ({escape(forge_url)}) is not valid!") + + +def _update_add_forge_request_info( + add_forge_request: AddForgeRequest, + task: Optional[Dict[str, Any]] = None, + task_run: Optional[Dict[str, Any]] = None, +) -> AddForgeRequestInfo: + """Update add forge request information out of the task and task_run information + + Args: + add_forge_request: Add request + task: Associated scheduler task information about the add request + task_run: Most recent run occurrence of the associated task + + Returns: + Summary of the add request information updated. + + """ + + # FIXME: Actually determine what'add_forge_request the update condition + must_add = False + if not add_forge_request.task_date or not add_forge_request.task_status: + if task: + must_add = True + add_forge_request.request_status = task["status"] + if task_run: + must_add = True + add_forge_request.task_date = task_run["ended"] + + if must_add: + add_forge_request.save() + + return add_forge_request.to_dict() + + +def create_add_forge_request( + forge_type: str, + forge_url: str, + privileged_user: bool = False, + user_id: Optional[int] = None, +) -> AddForgeRequestInfo: + """Create a loading task to add a software forge into the archive. + + This function aims to create a software forge listing task trough the use of the + swh-scheduler component. + + First, some checks are performed to see if the forge type and forge url are valid + but also if the the add request can be accepted. If those checks passed, the listing + task is then created. Otherwise, the add request is put in pending or rejected + state. + + All the submitted add requests are logged into the swh-web database to keep track of + them. + + Args: + forge_type: the type of forge to list (e.g. gitlab, cgit, ...) + forge_url: the url of the forge to add + privileged: Whether the user has some more privilege than other (bypass + review, access to privileged other forge types) + user_id: User identifier (provided when authenticated) + + Raises: + BadInputExc: the forge type or forge url is invalid or inexistent + ForbiddenExc: the provided forge url is blacklisted + + Returns: + dict: A dict describing the add request with the following keys: + + * **forge_type**: the type of visit to perform + * **forge_url**: the url of the forge + * **add_request_date**: the date the request was submitted + * **add_forge_request_status**: the request request_status, either + ****accepted**, rejected** or **pending** + * **add_task_status**: the forge loading task request_status, either + **not created**, **not yet scheduled**, **scheduled**, + **succeed** or **failed** + + """ + # FIXME: actually read this out of the scheduler *somehow* + forge_type_tasks = { + "sourceforge": "list-sourceforge-incremental", + "opam": "list-opam", + "gitlab": "list-gitlab-incremental", + "cgit": "list-cgit", + "launchpad": "list-launchpad-incremental", + } + _check_forge_type(forge_type, privileged_user) + _check_forge_url_valid(forge_url) + + print(f"################### forge_url {forge_url}") + print(f"################### forge_type {forge_type}") + # if all checks passed so far, we can try and add the forge + add_forge_request_status = can_add_forge(forge_url, privileged_user) + print(f"################### add_forge_request_status {add_forge_request_status}") + task = None + + # if the forge add request is accepted, create a scheduler + # task to load it into the archive + if add_forge_request_status == ADD_FORGE_REQUEST_ACCEPTED: + # create a task with high priority + task_kwargs: Dict[str, Any] = { + "url": forge_url, + } + + add_forge_request = None + # get list of previously submitted add requests (most recent first) + current_add_forge_requests = list( + AddForgeRequest.objects.filter( + forge_type=forge_type, forge_url=forge_url + ).order_by("-request_date") + ) + + # if no add requests already submitted, create the scheduler task + if not current_add_forge_requests: + task_dict = create_oneshot_task_dict( + forge_type_tasks[forge_type], **task_kwargs + ) + + task = scheduler().create_tasks([task_dict])[0] + + # pending add request has been accepted + if add_forge_request: + add_forge_request.request_status = ADD_FORGE_REQUEST_ACCEPTED + add_forge_request.task_id = task["id"] + add_forge_request.save() + else: + add_forge_request = AddForgeRequest.objects.create( + forge_type=forge_type, + forge_url=forge_url, + request_status=add_forge_request_status, + task_id=task["id"], + user_ids=f'"{user_id}"' if user_id else None, + ) + + # add request must be manually reviewed for acceptation + elif add_forge_request_status == ADD_FORGE_REQUEST_PENDING: + # check if there is already such a add request already submitted, + # no need to add it to the database in that case + try: + add_forge_request = AddForgeRequest.objects.get( + forge_type=forge_type, + forge_url=forge_url, + request_status=add_forge_request_status, + ) + user_ids = ( + add_forge_request.user_ids + if add_forge_request.user_ids is not None + else "" + ) + if user_id is not None and f'"{user_id}"' not in user_ids: + # update user ids list + add_forge_request.user_ids = f'{add_forge_request.user_ids},"{user_id}"' + add_forge_request.save() + + # if not add it to the database + except ObjectDoesNotExist: + add_forge_request = AddForgeRequest.objects.create( + forge_type=forge_type, + forge_url=forge_url, + request_status=add_forge_request_status, + user_ids=f'"{user_id}"' if user_id else None, + ) + # forge cannot be added as its url is blacklisted, + # log the request to the database anyway + else: + add_forge_request = AddForgeRequest.objects.create( + forge_type=forge_type, + forge_url=forge_url, + request_status=add_forge_request_status, + user_ids=f'"{user_id}"' if user_id else None, + ) + + if add_forge_request_status == ADD_FORGE_REQUEST_REJECTED: + raise ForbiddenExc( + ( + 'The "add forge now" request has been rejected ' + "because the provided forge url is blacklisted." + ) + ) + + assert add_forge_request is not None + return _update_add_forge_request_info(add_forge_request, task) + + +def _savable_forge_types(): + docstring = "" + if os.environ.get("DJANGO_SETTINGS_MODULE") != "swh.web.settings.tests": + forge_types = sorted(get_forge_types()) + docstring = "" + for forge_type in forge_types[:-1]: + docstring += f"**{forge_type}**, " + docstring += f"and **{forge_types[-1]}**" + return docstring + + +def update_add_forge_requests_from_queryset( + requests_queryset: QuerySet, +) -> List[AddForgeRequestInfo]: + """Update all add requests from a AddForgeRequest queryset, update their status in db + and return the list of impacted add_requests. + + Args: + requests_queryset: input AddForgeRequest queryset + + Returns: + A list of add forge request info dicts + + """ + task_ids = [] + for add_forge_query in requests_queryset: + task_ids.append(add_forge_query.task_id) + add_requests = [] + if task_ids: + try: + tasks = scheduler().get_tasks(task_ids) + tasks = {task["id"]: task for task in tasks} + task_runs = scheduler().get_task_runs(tasks) + task_runs = {task_run["task"]: task_run for task_run in task_runs} + except Exception: + # allow to avoid mocking api GET responses for /forge/add endpoint when + # running cypress tests as scheduler is not available + tasks = {} + task_runs = {} + for add_forge_query in requests_queryset: + sr_dict = _update_add_forge_request_info( + add_forge_query, + tasks.get(add_forge_query.task_id), + task_runs.get(add_forge_query.task_id), + ) + add_requests.append(sr_dict) + return add_requests + + +def get_add_forge_requests( + forge_type: str, forge_url: str +) -> List[AddForgeRequestInfo]: + """ + Get all add requests for a given software forge. + + Args: + forge_type: the type of visit + forge_url: the url of the forge + + Raises: + BadInputExc: the visit type or forge url is invalid + swh.web.common.exc.NotFoundExc: no add requests can be found for the + given forge + + Returns: + list: A list of add forge requests dict as described in + :func:`swh.web.common.forge_add.create_add_forge_request` + """ + _check_forge_type(forge_type) + _check_forge_url_valid(forge_url) + forges_query_set = AddForgeRequest.objects.filter( + forge_type=forge_type, forge_url=forge_url + ) + if forges_query_set.count() == 0: + raise NotFoundExc( + f"No add forge requests found on forge with url {forge_url}" + f"and type {forge_type}." + ) + return update_add_forge_requests_from_queryset(forges_query_set) + + +@api_route( + r"/forge/add/(?P.+)/url/(?P.+)/", + "api-1-add-forge", + methods=["GET", "POST"], + throttle_scope="swh_add_forge", + never_cache=True, +) +@api_doc("/forge/add/") +@format_docstring(forge_types=_savable_forge_types()) +def api_add_forge( + request, forge_type: str, forge_url: str +) -> List[AddForgeRequestInfo]: + """.. http:get:: /api/1/forge/add/(forge_type)/url/(forge_url)/ + .. http:post:: /api/1/forge/add/(forge_type)/url/(forge_url)/ + + Request adding a new software forge to list into the archive or check the + request_status of previously created add requests. + + That endpoint enables creating an add forge task for listing through a POST + request. + + Depending on the provided forge url, the add request can either be: + + * immediately **accepted**, for well known and supported forge type hosting + providers like for instance GitLab or Heptapod instance + * **rejected**, in case the url is blacklisted by Software Heritage + * **put in pending state** until a manual check is done in order to + determine if it can be listed or not + + Once a saving request has been accepted, its associated task request_status can + then be checked through a GET request on the same url. Returned request_status + can either be: + + * **not created**: no adding task has been created + * **not yet scheduled**: adding task has been created but its execution has + not yet been scheduled + * **scheduled**: the task execution has been scheduled + * **succeeded**: the task has been successfully executed + * **failed**: the task has been executed but it failed + + When issuing a POST request an object will be returned while a GET request will + return an array of objects (as multiple add requests might have been submitted + for the same forge). + + :param string forge_type: the type of forge to list (currently the supported + types are {forge_types}) + + :param string forge_url: the url of the forge to add + + {common_headers} + + :>json string forge_url: the url of the forge to add + :>json string forge_type: the type of forge + :>json string request_date: the date (in iso format) the add + request was issued + :>json string request_status: the request_status of the add request, + either **accepted**, **rejected** or **pending** + :>json string task_status: the request_status of the forge saving task, + either **not created**, **not yet scheduled**, **scheduled**, + **succeeded** or **failed** + :>json string note: optional note giving details about the add request, + for instance why it has been rejected + + :statuscode 200: no error + :statuscode 400: an invalid forge type or forge url has been provided + :statuscode 403: the provided forge url is blacklisted + :statuscode 404: no add requests have been found for a given forge + + """ + + if request.method == "POST": + add_forge_request = create_add_forge_request( + forge_type, + forge_url, + privileged_user( + request, + permissions=[SWH_AMBASSADOR_PERMISSION, API_ADD_FORGE_PERMISSION], + ), + user_id=request.user.id, + ) + # FIXME: drop the id of the result + # add_forge_request.pop("id", None) + result = [add_forge_request] + else: + add_forge_requests = get_add_forge_requests(forge_type, forge_url) + # FIXME: drop the id of the elements result + result = [add_forge_request for add_forge_request in add_forge_requests] + + print(f"############ result {result}") + return result diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py --- a/swh/web/auth/utils.py +++ b/swh/web/auth/utils.py @@ -17,6 +17,7 @@ SWH_AMBASSADOR_PERMISSION = "swh.ambassador" API_SAVE_ORIGIN_PERMISSION = "swh.web.api.save_origin" +API_ADD_FORGE_PERMISSION = "swh.web.api.add_forge" ADMIN_LIST_DEPOSIT_PERMISSION = "swh.web.admin.list_deposits" diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -46,6 +46,10 @@ return redirect(reverse("origin-save")) +def _browse_forge_add_view(request): + return redirect(reverse("forge-add")) + + urlpatterns = [ url(r"^$", _browse_search_view), url(r"^help/$", _browse_help_view, name="browse-help"), @@ -58,6 +62,7 @@ swhid_browse, name="browse-swhid", ), + url(r"^forge/add/$", _browse_forge_add_view, name="browse-forge-add"), ] urlpatterns += BrowseUrls.get_url_patterns() diff --git a/swh/web/common/forge_add.py b/swh/web/common/forge_add.py new file mode 100644 --- /dev/null +++ b/swh/web/common/forge_add.py @@ -0,0 +1,22 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import List + + +def get_forge_types(privileged_user: bool = False) -> List[str]: + """Retrieve the allowed list of forge types users can request. + + Args: + privileged_user: Whether the user is privileged (True) or not (False). + + Returns: + The list of supported forge to request for listing. + + """ + # FIXME: Determine where to store such information + hardcoded_types = ["gitlab", "heptapod", "cgit", "launchpad", "opam"] + hardcoded_types.sort() + return hardcoded_types diff --git a/swh/web/common/migrations/0013_auto_20211119_1031.py b/swh/web/common/migrations/0013_auto_20211119_1031.py new file mode 100644 --- /dev/null +++ b/swh/web/common/migrations/0013_auto_20211119_1031.py @@ -0,0 +1,108 @@ +# Generated by Django 2.2.24 on 2021-11-19 10:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("swh_web_common", "0012_saveoriginrequest_note"), + ] + + operations = [ + migrations.CreateModel( + name="AddAuthorizedForge", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("url", models.CharField(max_length=200)), + ], + options={"db_table": "add_authorized_forge",}, + ), + migrations.CreateModel( + name="AddForgeRequest", + fields=[ + ("id", models.BigAutoField(primary_key=True, serialize=False)), + ("request_date", models.DateTimeField(auto_now_add=True)), + ("forge_type", models.CharField(max_length=200)), + ("forge_url", models.CharField(max_length=200)), + ( + "request_status", + models.TextField( + choices=[ + ("accepted", "accepted"), + ("rejected", "rejected"), + ("pending", "pending"), + ], + default="pending", + ), + ), + ("user_ids", models.TextField(null=True)), + ("note", models.TextField(null=True)), + ], + options={"db_table": "add_forge_request", "ordering": ["-id"],}, + ), + migrations.CreateModel( + name="AddUnauthorizedForge", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("url", models.CharField(max_length=200)), + ], + options={"db_table": "add_unauthorized_forge",}, + ), + migrations.AddIndex( + model_name="addunauthorizedforge", + index=models.Index(fields=["url"], name="add_unautho_url_c89686_idx"), + ), + migrations.AddIndex( + model_name="addforgerequest", + index=models.Index( + fields=["forge_url", "request_status"], + name="add_forge_r_forge_u_3703b5_idx", + ), + ), + migrations.AddIndex( + model_name="addauthorizedforge", + index=models.Index(fields=["url"], name="add_authori_url_dd17ed_idx"), + ), + migrations.AddField( + model_name="addforgerequest", + name="task_date", + field=models.DateTimeField(null=True), + ), + migrations.AddField( + model_name="addforgerequest", + name="task_id", + field=models.IntegerField(default=-1), + ), + migrations.AddField( + model_name="addforgerequest", + name="task_status", + field=models.TextField( + choices=[ + ("not created", "not created"), + ("not yet scheduled", "not yet scheduled"), + ("scheduled", "scheduled"), + ("succeeded", "succeeded"), + ("failed", "failed"), + ("running", "running"), + ], + default="not created", + ), + ), + ] diff --git a/swh/web/common/models.py b/swh/web/common/models.py --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -5,7 +5,7 @@ from django.db import models -from swh.web.common.typing import SaveOriginRequestInfo +from swh.web.common.typing import AddForgeRequestInfo, SaveOriginRequestInfo class SaveAuthorizedOrigin(models.Model): @@ -133,3 +133,106 @@ def __str__(self) -> str: return str(self.to_dict()) + + +class AddAuthorizedForge(models.Model): + """Model table holding authorized forge urls to be loaded into the archive. + + """ + + url = models.CharField(max_length=200, null=False) + + class Meta: + app_label = "swh_web_common" + db_table = "add_authorized_forge" + indexes = [models.Index(fields=["url"])] + + def __str__(self): + return self.url + + +class AddUnauthorizedForge(models.Model): + """Model table holding unauthorized forge urls to be loaded into the archive. + + """ + + url = models.CharField(max_length=200, null=False) + + class Meta: + app_label = "swh_web_common" + db_table = "add_unauthorized_forge" + indexes = [models.Index(fields=["url"])] + + def __str__(self): + return self.url + + +ADD_FORGE_REQUEST_ACCEPTED = "accepted" +ADD_FORGE_REQUEST_REJECTED = "rejected" +ADD_FORGE_REQUEST_PENDING = "pending" + +ADD_FORGE_REQUEST_STATUS = [ + (ADD_FORGE_REQUEST_ACCEPTED, ADD_FORGE_REQUEST_ACCEPTED), + (ADD_FORGE_REQUEST_REJECTED, ADD_FORGE_REQUEST_REJECTED), + (ADD_FORGE_REQUEST_PENDING, ADD_FORGE_REQUEST_PENDING), +] + +ADD_FORGE_TASK_NOT_CREATED = "not created" +ADD_FORGE_TASK_NOT_YET_SCHEDULED = "not yet scheduled" +ADD_FORGE_TASK_SCHEDULED = "scheduled" +ADD_FORGE_TASK_SUCCEEDED = "succeeded" +ADD_FORGE_TASK_FAILED = "failed" +ADD_FORGE_TASK_RUNNING = "running" + +ADD_FORGE_TASK_STATUS = SAVE_TASK_STATUS + + +class AddForgeRequest(models.Model): + """The "add forge" users issued requests table. + + """ + + id = models.BigAutoField(primary_key=True) + request_date = models.DateTimeField(auto_now_add=True) + forge_type = models.CharField(max_length=200, null=False) + forge_url = models.CharField(max_length=200, null=False) + request_status = models.TextField( + choices=ADD_FORGE_REQUEST_STATUS, default=ADD_FORGE_REQUEST_PENDING + ) + task_id = models.IntegerField(default=-1) + task_date = models.DateTimeField(null=True) + task_status = models.TextField( + choices=ADD_FORGE_TASK_STATUS, default=ADD_FORGE_TASK_NOT_CREATED + ) + + # store ids of users that submitted the request as string list + user_ids = models.TextField(null=True) + note = models.TextField(null=True) + + class Meta: + app_label = "swh_web_common" + db_table = "add_forge_request" + ordering = ["-id"] + indexes = [models.Index(fields=["forge_url", "request_status"])] + + def to_dict(self) -> AddForgeRequestInfo: + """Map the request save model object to a json serializable dict. + + Returns: + The corresponding AddForgeRequestInfo json serializable dict. + + """ + return AddForgeRequestInfo( + id=self.id, + forge_url=self.forge_url, + forge_type=self.forge_type, + request_date=self.request_date.isoformat(), + request_status=self.request_status, + task_id=self.task_id, + task_date=self.task_date.isoformat() if self.task_date else None, + task_status=self.task_status, + note=self.note, + ) + + def __str__(self) -> str: + return str(self.to_dict()) diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -261,3 +261,24 @@ """content length of the artifact""" last_modified: Optional[str] """Last modification time reported by the server (as iso8601 string)""" + + +class AddForgeRequestInfo(TypedDict): + id: int + """Unique key""" + request_date: str + """Date of the creation request""" + forge_type: str + """Type of the forge""" + request_status: Optional[str] + """Status of the request""" + forge_url: str + """Forge to list""" + task_id: Optional[int] + """Identifier of the loading task in the scheduler if scheduled""" + task_date: Optional[str] + """End of the listing if terminated""" + task_status: Optional[str] + """Status of the scheduled task""" + note: Optional[str] + """Optional note associated to the request, for instance rejection reason""" diff --git a/swh/web/misc/forge_add.py b/swh/web/misc/forge_add.py new file mode 100644 --- /dev/null +++ b/swh/web/misc/forge_add.py @@ -0,0 +1,114 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Any, Dict + +from django.conf.urls import url +from django.core.paginator import Paginator +from django.db.models import Q +from django.http import JsonResponse +from django.shortcuts import render + +from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION, privileged_user +from swh.web.common.forge_add import get_forge_types +from swh.web.common.models import AddForgeRequest + + +def get_add_forge_task_info( + add_forge_request_id: int, full_info: bool = True +) -> Dict[str, Any]: + """Get detailed information about an accepted add forge request and its associated + loading task. + + """ + return {} + + +def _forge_add_view(request): + return render( + request, + "misc/forge-add.html", + { + "heading": ( + "Request adding a software forge to list and ingest in the archive" + ), + "forge_types": get_forge_types( + privileged_user(request, permissions=[SWH_AMBASSADOR_PERMISSION]) + ), + }, + ) + + +def _forge_add_requests_list(request, status: str) -> JsonResponse: + + if status != "all": + add_requests = AddForgeRequest.objects.filter(request_status=status) + else: + add_requests = AddForgeRequest.objects.all() + + print("########### request", request) + print("########### status", status) + print("########### add_requests", add_requests) + table_data = {} + table_data["recordsTotal"] = add_requests.count() + table_data["draw"] = int(request.GET["draw"]) + + search_value = request.GET["search[value]"] + + column_order = request.GET["order[0][column]"] + field_order = request.GET["columns[%s][name]" % column_order] + order_dir = request.GET["order[0][dir]"] + if order_dir == "desc": + field_order = "-" + field_order + + add_requests = add_requests.order_by(field_order) + + length = int(request.GET["length"]) + page = int(int(request.GET["start"]) / length + 1) + + if search_value: + add_requests = add_requests.filter( + Q(status__icontains=search_value) + | Q(task_status__icontains=search_value) + | Q(forge_type__icontains=search_value) + | Q(forge_url__icontains=search_value) + ) + + if ( + int(request.GET.get("user_requests_only", "0")) + and request.user.is_authenticated + ): + add_requests = add_requests.filter(user_ids__contains=f'"{request.user.id}"') + + table_data["recordsFiltered"] = add_requests.count() + paginator = Paginator(add_requests, length) + data = [sor.to_dict() for sor in paginator.page(page).object_list] + table_data["data"] = data # type: ignore + return JsonResponse(table_data) + + +def _add_forge_task_info(request, add_request_id): + request_info = get_add_forge_task_info( + add_request_id, full_info=request.user.is_staff + ) + for date_field in ("scheduled", "started", "ended"): + if date_field in request_info and request_info[date_field] is not None: + request_info[date_field] = request_info[date_field].isoformat() + return JsonResponse(request_info) + + +urlpatterns = [ + url(r"^add/$", _forge_add_view, name="forge-add"), + url( + r"^add/requests/list/(?P.+)/$", + _forge_add_requests_list, + name="forge-add-requests-list", + ), + url( + r"^add/task/info/(?P.+)/", + _add_forge_task_info, + name="forge-add-task-info", + ), +] diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -52,6 +52,7 @@ url(r"^", include("swh.web.misc.badges")), url(r"^metrics/prometheus/$", prometheus_metrics, name="metrics-prometheus"), url(r"^", include("swh.web.misc.iframe")), + url(r"^", include("swh.web.misc.forge_add")), ] diff --git a/swh/web/templates/admin/forge-add.html b/swh/web/templates/admin/forge-add.html new file mode 100644 --- /dev/null +++ b/swh/web/templates/admin/forge-add.html @@ -0,0 +1,186 @@ +{% extends "layout.html" %} + +{% comment %} +Copyright (C) 2021 The Software Heritage developers +See the AUTHORS file at the top-level directory of this distribution +License: GNU Affero General Public License version 3, or any later version +See top-level LICENSE file for more information +{% endcomment %} + +{% load swh_templatetags %} +{% load render_bundle from webpack_loader %} + +{% block header %} +{{ block.super }} +{% render_bundle 'admin' %} +{% render_bundle 'forge_add' %} +{% endblock %} + +{% block title %} "Add forge" administration {% endblock %} + +{% block navbar-content %} +

Add forge administration

+{% endblock %} + +{% block content %} + + + +
+ +
+ +
+
+ + + + + + + + + +
DateTypeUrl
+
+
+
+
+ +
+
+ +
+
+ +
+
+
+
+ + + + + + + + + + + +
DateTypeUrlStatusInfo
+
+
+
+
+ +
+
+
+
+ + + + + + + + + + +
DateTypeUrlInfo
+
+
+
+
+ +
+
+
+
+
+ +
+ + +
+
+ + + + + + +
Url
+
+
+
+ +
+ +
+
+
+
+ +
+
+
+
+
+ +
+ + + + + + +
Url
+
+
+
+ +
+ +
+
+
+
+ +
+
+
+
+
+
+
+
+ + +{% endblock %} diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -208,6 +208,12 @@

Save code now

+ + {% endif %}