diff --git a/swh/web/assets/src/bundles/admin/origin-save.js b/swh/web/assets/src/bundles/admin/origin-save.js index 3f60c6df..b595fe0b 100644 --- a/swh/web/assets/src/bundles/admin/origin-save.js +++ b/swh/web/assets/src/bundles/admin/origin-save.js @@ -1,313 +1,313 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost} from 'utils/functions'; let authorizedOriginTable; let unauthorizedOriginTable; let pendingSaveRequestsTable; let acceptedSaveRequestsTable; let rejectedSaveRequestsTable; function enableRowSelection(tableSel) { $(`${tableSel} tbody`).on('click', 'tr', function() { if ($(this).hasClass('selected')) { $(this).removeClass('selected'); } else { $(`${tableSel} tr.selected`).removeClass('selected'); $(this).addClass('selected'); } }); } export function initOriginSaveAdmin() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'throw'; authorizedOriginTable = $('#swh-authorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_authorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-authorized-origin-urls'); unauthorizedOriginTable = $('#swh-unauthorized-origin-urls').DataTable({ serverSide: true, ajax: Urls.admin_origin_save_unauthorized_urls_list(), columns: [{data: 'url', name: 'url'}], scrollY: '50vh', scrollCollapse: true, info: false }); enableRowSelection('#swh-unauthorized-origin-urls'); let columnsData = [ { data: 'id', name: 'id', visible: false, searchable: false }, { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { data: 'origin_type', name: 'origin_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } } ]; pendingSaveRequestsTable = $('#swh-origin-save-pending-requests').DataTable({ serverSide: true, - ajax: Urls.browse_origin_save_requests_list('pending'), + ajax: Urls.origin_save_requests_list('pending'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-pending-requests'); rejectedSaveRequestsTable = $('#swh-origin-save-rejected-requests').DataTable({ serverSide: true, - ajax: Urls.browse_origin_save_requests_list('rejected'), + ajax: Urls.origin_save_requests_list('rejected'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-rejected-requests'); columnsData.push({ data: 'save_task_status', name: 'save_task_status', render: (data, type, row) => { if (data === 'succeed') { let browseOriginUrl = Urls.browse_origin(row.origin_url); return `${data}`; } return data; } }); acceptedSaveRequestsTable = $('#swh-origin-save-accepted-requests').DataTable({ serverSide: true, - ajax: Urls.browse_origin_save_requests_list('accepted'), + ajax: Urls.origin_save_requests_list('accepted'), searchDelay: 1000, columns: columnsData, scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); enableRowSelection('#swh-origin-save-accepted-requests'); $('#swh-origin-save-requests-nav-item').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-origin-save-url-filters-nav-item').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-authorized-origins-tab').on('shown.bs.tab', () => { authorizedOriginTable.draw(); }); $('#swh-unauthorized-origins-tab').on('shown.bs.tab', () => { unauthorizedOriginTable.draw(); }); $('#swh-save-requests-pending-tab').on('shown.bs.tab', () => { pendingSaveRequestsTable.draw(); }); $('#swh-save-requests-accepted-tab').on('shown.bs.tab', () => { acceptedSaveRequestsTable.draw(); }); $('#swh-save-requests-rejected-tab').on('shown.bs.tab', () => { rejectedSaveRequestsTable.draw(); }); $('#swh-save-requests-pending-tab').click(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-accepted-tab').click(() => { acceptedSaveRequestsTable.ajax.reload(null, false); }); $('#swh-save-requests-rejected-tab').click(() => { rejectedSaveRequestsTable.ajax.reload(null, false); }); }); } export function addAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_authorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row.add({'url': originUrl}).draw(); }) .catch(response => { swh.webapp.showModalMessage( 'Duplicated origin url prefix', 'The provided origin url prefix is already registered in the authorized list.'); }); } export function removeAuthorizedOriginUrl() { let originUrl = $('#swh-authorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_authorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { authorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function addUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-url-prefix').val(); let addOriginUrl = Urls.admin_origin_save_add_unauthorized_url(originUrl); csrfPost(addOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row.add({'url': originUrl}).draw(); }) .catch(() => { swh.webapp.showModalMessage( 'Duplicated origin url prefix', 'The provided origin url prefix is already registered in the unauthorized list.'); }); } export function removeUnauthorizedOriginUrl() { let originUrl = $('#swh-unauthorized-origin-urls tr.selected').text(); if (originUrl) { let removeOriginUrl = Urls.admin_origin_save_remove_unauthorized_url(originUrl); csrfPost(removeOriginUrl) .then(handleFetchError) .then(() => { unauthorizedOriginTable.row('.selected').remove().draw(); }) .catch(() => {}); } } export function acceptOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let acceptOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); let acceptSaveRequestUrl = Urls.admin_origin_save_request_accept(rowData['origin_type'], rowData['origin_url']); csrfPost(acceptSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Accept origin save request ?', 'Are you sure to accept this origin save request ?', acceptOriginSaveRequestCallback); } } export function rejectOriginSaveRequest() { let selectedRow = pendingSaveRequestsTable.row('.selected'); if (selectedRow.length) { let rejectOriginSaveRequestCallback = () => { let rowData = selectedRow.data(); let rejectSaveRequestUrl = Urls.admin_origin_save_request_reject(rowData['origin_type'], rowData['origin_url']); csrfPost(rejectSaveRequestUrl) .then(() => { pendingSaveRequestsTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Reject origin save request ?', 'Are you sure to reject this origin save request ?', rejectOriginSaveRequestCallback); } } function removeOriginSaveRequest(requestTable) { let selectedRow = requestTable.row('.selected'); if (selectedRow.length) { let requestId = selectedRow.data()['id']; let removeOriginSaveRequestCallback = () => { let removeSaveRequestUrl = Urls.admin_origin_save_request_remove(requestId); csrfPost(removeSaveRequestUrl) .then(() => { requestTable.ajax.reload(null, false); }); }; swh.webapp.showModalConfirm( 'Remove origin save request ?', 'Are you sure to remove this origin save request ?', removeOriginSaveRequestCallback); } } export function removePendingOriginSaveRequest() { removeOriginSaveRequest(pendingSaveRequestsTable); } export function removeAcceptedOriginSaveRequest() { removeOriginSaveRequest(acceptedSaveRequestsTable); } export function removeRejectedOriginSaveRequest() { removeOriginSaveRequest(rejectedSaveRequestsTable); } diff --git a/swh/web/assets/src/bundles/browse/index.js b/swh/web/assets/src/bundles/browse/index.js index 751ecc3a..4ab79b40 100644 --- a/swh/web/assets/src/bundles/browse/index.js +++ b/swh/web/assets/src/bundles/browse/index.js @@ -1,19 +1,18 @@ /** * Copyright (C) 2018 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ // main bundle for the swh-web/browse application import './browse.css'; import './breadcrumbs.css'; import './content.css'; import './snapshot-navigation.css'; export * from './snapshot-navigation'; export * from './origin-search'; export * from './browse-utils'; export * from './swh-ids-utils'; -export * from './origin-save'; diff --git a/swh/web/assets/src/bundles/browse/origin-save.js b/swh/web/assets/src/bundles/save/index.js similarity index 98% rename from swh/web/assets/src/bundles/browse/origin-save.js rename to swh/web/assets/src/bundles/save/index.js index c1578c14..196bdbeb 100644 --- a/swh/web/assets/src/bundles/browse/origin-save.js +++ b/swh/web/assets/src/bundles/save/index.js @@ -1,299 +1,299 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost, isGitRepoUrl, removeUrlFragment} from 'utils/functions'; import {validate} from 'validate.js'; let saveRequestsTable; function originSaveRequest(originType, originUrl, acceptedCallback, pendingCallback, errorCallback) { - let addSaveOriginRequestUrl = Urls.browse_origin_save_request(originType, originUrl); + let addSaveOriginRequestUrl = Urls.origin_save_request(originType, originUrl); let headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' }; $('.swh-processing-save-request').css('display', 'block'); csrfPost(addSaveOriginRequestUrl, headers) .then(handleFetchError) .then(response => response.json()) .then(data => { $('.swh-processing-save-request').css('display', 'none'); if (data.save_request_status === 'accepted') { acceptedCallback(); } else { pendingCallback(); } }) .catch(response => { $('.swh-processing-save-request').css('display', 'none'); errorCallback(response.status); }); } function htmlAlert(type, message) { return ``; } export function initOriginSave() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'none'; - fetch(Urls.browse_origin_save_types_list()) + fetch(Urls.origin_save_types_list()) .then(response => response.json()) .then(data => { for (let originType of data) { $('#swh-input-origin-type').append(``); } }); saveRequestsTable = $('#swh-origin-save-requests') .on('error.dt', (e, settings, techNote, message) => { $('#swh-origin-save-request-list-error').text('An error occurred while retrieving the save requests list'); console.log(message); }) .DataTable({ serverSide: true, - ajax: Urls.browse_origin_save_requests_list('all'), + ajax: Urls.origin_save_requests_list('all'), searchDelay: 1000, columns: [ { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { data: 'origin_type', name: 'origin_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } }, { data: 'save_request_status', name: 'status' }, { data: 'save_task_status', name: 'loading_task_status', render: (data, type, row) => { if (data === 'succeed') { let browseOriginUrl = Urls.browse_origin(row.origin_url); if (row.visit_date) { browseOriginUrl += `visit/${row.visit_date}/`; } return `${data}`; } return data; } } ], scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); $('#swh-origin-save-requests-list-tab').on('shown.bs.tab', () => { saveRequestsTable.draw(); window.location.hash = '#requests'; }); $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => { removeUrlFragment(); }); let saveRequestAcceptedAlert = htmlAlert( 'success', 'The "save code now" request has been accepted and will be processed as soon as possible.' ); let saveRequestPendingAlert = htmlAlert( 'warning', 'The "save code now" request has been put in pending state and may be accepted for processing after manual review.' ); let saveRequestRejectedAlert = htmlAlert( 'danger', 'The "save code now" request has been rejected because the provided origin url is blacklisted.' ); let saveRequestRateLimitedAlert = htmlAlert( 'danger', 'The rate limit for "save code now" requests has been reached. Please try again later.' ); let saveRequestUnknownErrorAlert = htmlAlert( 'danger', 'An unexpected error happened when submitting the "save code now request' ); $('#swh-save-origin-form').submit(event => { event.preventDefault(); event.stopPropagation(); $('.alert').alert('close'); if (event.target.checkValidity()) { $(event.target).removeClass('was-validated'); let originType = $('#swh-input-origin-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode) => { $('#swh-origin-save-request-status').css('color', 'red'); if (statusCode === 403) { $('#swh-origin-save-request-status').html(saveRequestRejectedAlert); } else if (statusCode === 429) { $('#swh-origin-save-request-status').html(saveRequestRateLimitedAlert); } else { $('#swh-origin-save-request-status').html(saveRequestUnknownErrorAlert); } }); } else { $(event.target).addClass('was-validated'); } }); $('#swh-show-origin-save-requests-list').on('click', (event) => { event.preventDefault(); $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); }); $('#swh-input-origin-url').on('input', function(event) { let originUrl = $(this).val().trim(); $(this).val(originUrl); $('#swh-input-origin-type option').each(function() { let val = $(this).val(); if (val && originUrl.includes(val)) { $(this).prop('selected', true); } }); }); if (window.location.hash === '#requests') { $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); } }); } export function validateSaveOriginUrl(input) { let originUrl = input.value.trim(); let validUrl = validate({website: originUrl}, { website: { url: { schemes: ['http', 'https', 'svn', 'git'] } } }) === undefined; let originType = $('#swh-input-origin-type').val(); if (originType === 'git' && validUrl) { // additional checks for well known code hosting providers let githubIdx = originUrl.indexOf('://github.com'); let gitlabIdx = originUrl.indexOf('://gitlab.'); let gitSfIdx = originUrl.indexOf('://git.code.sf.net'); let bitbucketIdx = originUrl.indexOf('://bitbucket.org'); if (githubIdx !== -1 && githubIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'github.com'); } else if (gitlabIdx !== -1 && gitlabIdx <= 5) { let startIdx = gitlabIdx + 3; let idx = originUrl.indexOf('/', startIdx); if (idx !== -1) { let gitlabDomain = originUrl.substr(startIdx, idx - startIdx); // GitLab repo url needs to be suffixed by '.git' in order to be successfully loaded // This is due to a bug in dulwich < 0.19.11. // TODO: remove this check once dulwich >= 0.19.11 is used in production validUrl = isGitRepoUrl(originUrl, gitlabDomain) && originUrl.endsWith('.git'); } else { validUrl = false; } } else if (gitSfIdx !== -1 && gitSfIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'git.code.sf.net/p'); } else if (bitbucketIdx !== -1 && bitbucketIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'bitbucket.org'); } } if (validUrl) { input.setCustomValidity(''); } else { input.setCustomValidity('The origin url is not valid or does not reference a code repository'); } } export function initTakeNewSnapshot() { let newSnapshotRequestAcceptedAlert = htmlAlert( 'success', 'The "take new snapshot" request has been accepted and will be processed as soon as possible.' ); let newSnapshotRequestPendingAlert = htmlAlert( 'warning', 'The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.' ); let newSnapshotRequestRejectedAlert = htmlAlert( 'danger', 'The "take new snapshot" request has been rejected.' ); let newSnapshotRequestRateLimitAlert = htmlAlert( 'danger', 'The rate limit for "take new snapshot" requests has been reached. Please try again later.' ); let newSnapshotRequestUnknownErrorAlert = htmlAlert( 'danger', 'An unexpected error happened when submitting the "save code now request".' ); $(document).ready(() => { $('#swh-take-new-snapshot-form').submit(event => { event.preventDefault(); event.stopPropagation(); let originType = $('#swh-input-origin-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode) => { $('#swh-take-new-snapshot-request-status').css('color', 'red'); if (statusCode === 403) { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRejectedAlert); } else if (statusCode === 429) { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRateLimitAlert); } else { $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestUnknownErrorAlert); } }); }); }); } diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py index ace5caf4..ea06eb42 100644 --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -1,53 +1,52 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url -from django.shortcuts import render +from django.shortcuts import render, redirect import swh.web.browse.views.directory # noqa import swh.web.browse.views.content # noqa -import swh.web.browse.views.origin_save # noqa import swh.web.browse.views.origin # noqa import swh.web.browse.views.person # noqa import swh.web.browse.views.release # noqa import swh.web.browse.views.revision # noqa import swh.web.browse.views.snapshot # noqa from swh.web.browse.browseurls import BrowseUrls from swh.web.browse.identifiers import swh_id_browse +from swh.web.common.utils import reverse def _browse_help_view(request): return render(request, 'browse/help.html', {'heading': 'How to browse the archive ?'}) def _browse_search_view(request): return render(request, 'browse/search.html', {'heading': 'Search software origins to browse'}) def _browse_vault_view(request): return render(request, 'browse/vault-ui.html', {'heading': 'Download archive content from the Vault'}) def _browse_origin_save_view(request): - return render(request, 'browse/origin-save.html', - {'heading': 'Request the saving of a software origin into the archive'}) # noqa + return redirect(reverse('origin-save')) urlpatterns = [ url(r'^$', _browse_search_view), url(r'^help/$', _browse_help_view, name='browse-help'), url(r'^search/$', _browse_search_view, name='browse-search'), url(r'^vault/$', _browse_vault_view, name='browse-vault'), + # for backward compatibility url(r'^origin/save/$', _browse_origin_save_view, name='browse-origin-save'), - # for backward compatibility - url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse) + url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse), ] urlpatterns += BrowseUrls.get_url_patterns() diff --git a/swh/web/browse/views/origin_save.py b/swh/web/misc/origin_save.py similarity index 78% rename from swh/web/browse/views/origin_save.py rename to swh/web/misc/origin_save.py index 66058117..818c2f71 100644 --- a/swh/web/browse/views/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -1,90 +1,102 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json +from django.conf.urls import url from django.core.paginator import Paginator from django.http import HttpResponse, HttpResponseForbidden +from django.shortcuts import render from rest_framework.decorators import api_view, authentication_classes -from swh.web.browse.browseurls import browse_route from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( create_save_origin_request, get_savable_origin_types, get_save_origin_requests_from_queryset ) from swh.web.common.throttling import throttle_scope from swh.web.common.utils import EnforceCSRFAuthentication -@browse_route(r'origin/save/(?P.+)/url/(?P.+)/', - view_name='browse-origin-save-request') +def _origin_save_view(request): + return render(request, 'misc/origin-save.html', + {'heading': ('Request the saving of a software origin into ' + 'the archive')}) + + @api_view(['POST']) @authentication_classes((EnforceCSRFAuthentication, )) @throttle_scope('swh_save_origin') -def _browse_origin_save_request(request, origin_type, origin_url): +def _origin_save_request(request, origin_type, origin_url): """ This view is called through AJAX from the save code now form of swh-web. We use DRF here as we want to rate limit the number of submitted requests per user to avoid being possibly flooded by bots. """ try: response = json.dumps(create_save_origin_request(origin_type, origin_url), separators=(',', ': ')) return HttpResponse(response, content_type='application/json') except ForbiddenExc as exc: return HttpResponseForbidden(str(exc)) -@browse_route(r'origin/save/types/list/', - view_name='browse-origin-save-types-list') -def _browse_origin_save_types_list(request): +def _origin_save_types_list(request): origin_types = json.dumps(get_savable_origin_types(), separators=(',', ': ')) return HttpResponse(origin_types, content_type='application/json') -@browse_route(r'origin/save/requests/list/(?P.+)/', - view_name='browse-origin-save-requests-list') -def _browse_origin_save_requests_list(request, status): +def _origin_save_requests_list(request, status): if status != 'all': save_requests = SaveOriginRequest.objects.filter(status=status) else: save_requests = SaveOriginRequest.objects.all() table_data = {} table_data['recordsTotal'] = save_requests.count() table_data['draw'] = int(request.GET['draw']) search_value = request.GET['search[value]'] column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order save_requests = save_requests.order_by(field_order) length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 save_requests = get_save_origin_requests_from_queryset(save_requests) if search_value: save_requests = \ [sr for sr in save_requests if search_value.lower() in sr['save_request_status'].lower() or search_value.lower() in sr['save_task_status'].lower() or search_value.lower() in sr['origin_type'].lower() or search_value.lower() in sr['origin_url'].lower()] table_data['recordsFiltered'] = len(save_requests) paginator = Paginator(save_requests, length) table_data['data'] = paginator.page(page).object_list table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') + + +urlpatterns = [ + url(r'^save/$', _origin_save_view, name='origin-save'), + url(r'^save/(?P.+)/url/(?P.+)/$', + _origin_save_request, name='origin-save-request'), + url(r'^save/types/list/$', _origin_save_types_list, + name='origin-save-types-list'), + url(r'^save/requests/list/(?P.+)/$', _origin_save_requests_list, + name='origin-save-requests-list'), +] diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py index 8455da81..83f3f407 100644 --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -1,57 +1,58 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.conf.urls import url, include from django.contrib.staticfiles import finders from django.shortcuts import render from swh.web.config import get_config def _jslicenses(request): jslicenses_file = finders.find('jssources/jslicenses.json') jslicenses_data = json.load(open(jslicenses_file)) jslicenses_data = sorted(jslicenses_data.items(), key=lambda item: item[0].split('/')[-1]) return render(request, "misc/jslicenses.html", {'jslicenses_data': jslicenses_data}) urlpatterns = [ url(r'^', include('swh.web.misc.coverage')), url(r'^jslicenses/$', _jslicenses, name='jslicenses'), + url(r'^', include('swh.web.misc.origin_save')), ] # when running end to end tests trough cypress, declare some extra # endpoints to provide input data for some of those tests if get_config()['e2e_tests_mode']: from swh.web.tests.data import ( get_content_code_data_by_ext, get_content_other_data_by_ext, get_content_code_data_all_exts, get_content_code_data_by_filename, get_content_code_data_all_filenames, ) # noqa urlpatterns.append( url(r'^tests/data/content/code/extension/(?P.+)/$', get_content_code_data_by_ext, name='tests-content-code-extension')) urlpatterns.append( url(r'^tests/data/content/other/extension/(?P.+)/$', get_content_other_data_by_ext, name='tests-content-other-extension')) urlpatterns.append(url(r'^tests/data/content/code/extensions/$', get_content_code_data_all_exts, name='tests-content-code-extensions')) urlpatterns.append( url(r'^tests/data/content/code/filename/(?P.+)/$', get_content_code_data_by_filename, name='tests-content-code-filename')) urlpatterns.append(url(r'^tests/data/content/code/filenames/$', get_content_code_data_all_filenames, name='tests-content-code-filenames')) diff --git a/swh/web/templates/browse/layout.html b/swh/web/templates/browse/layout.html index 688ed544..926734bb 100644 --- a/swh/web/templates/browse/layout.html +++ b/swh/web/templates/browse/layout.html @@ -1,23 +1,24 @@ {% extends "layout.html" %} {% comment %} Copyright (C) 2017-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% load render_bundle from webpack_loader %} {% block title %}{{ heading }} – Software Heritage archive {% endblock %} {% block header %} {% render_bundle 'browse' %} {% render_bundle 'vault' %} +{% render_bundle 'save' %} {% endblock %} {% block content %}
Beta version
{% block browse-content %}{% endblock %} {% endblock %} diff --git a/swh/web/templates/includes/take-new-snapshot.html b/swh/web/templates/includes/take-new-snapshot.html index 376f1ece..5b4ffe8a 100644 --- a/swh/web/templates/includes/take-new-snapshot.html +++ b/swh/web/templates/includes/take-new-snapshot.html @@ -1,76 +1,76 @@ {% comment %} Copyright (C) 2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% load swh_templatetags %} {% if snapshot_context and snapshot_context.origin_info and snapshot_context.origin_info.type|origin_type_savable %} {% endif %} \ No newline at end of file diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index e2b25e6b..65ddea80 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,213 +1,213 @@ {% comment %} Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block title %}{% endblock %} {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% block header %}{% endblock %}
{% block content %}{% endblock %}
{% include "includes/global-modals.html" %}
back to top
diff --git a/swh/web/templates/browse/origin-save.html b/swh/web/templates/misc/origin-save.html similarity index 93% rename from swh/web/templates/browse/origin-save.html rename to swh/web/templates/misc/origin-save.html index 4f0d3b52..21c4ec32 100644 --- a/swh/web/templates/browse/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,113 +1,120 @@ -{% extends "./layout.html" %} +{% extends "../layout.html" %} {% comment %} Copyright (C) 2018-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} +{% load render_bundle from webpack_loader %} {% load static %} +{% block title %}{{ heading }} – Software Heritage archive{% endblock %} + +{% block header %} +{% render_bundle 'save' %} +{% endblock %} + {% block navbar-content %}

Save code now

{% endblock %} -{% block browse-content %} +{% block content %}

You can contribute to extend the content of the Software Heritage archive by submitting an origin save request. To do so, fill the required info in the form below:

  • Origin type: the type of version control system the software origin is using. Currently, the supported types are:
  • Origin url: the url of the remote repository for the software origin.
    In order to avoid saving errors from Software Heritage, you should provide the clone/checkout url as given by the provider hosting the software origin.
    It can easily be found in the web interface used to browse the software origin.
    For instance, if you want to save a git origin into the archive, you should check that the command $ git clone <origin_url>
    does not return an error before submitting a request.

Once submitted, your save request can either be:

  • accepted: a visit to the provided origin will then be scheduled by Software Heritage in order to load its content into the archive as soon as possible
  • rejected: the provided origin url is blacklisted and no visit will be scheduled
  • put in pending state: a manual review will then be performed in order to determine if the origin can be safely loaded or not into the archive

Once a save request has been accepted, you can follow its current status in the submitted save requests list.

{% csrf_token %}
The origin type must be specified
- +
The origin url is not valid or does not reference a code repository
Date Type Url Request Status

{% endblock %} \ No newline at end of file diff --git a/swh/web/tests/misc/__init__.py b/swh/web/tests/misc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/tests/browse/views/test_origin_save.py b/swh/web/tests/misc/test_origin_save.py similarity index 78% rename from swh/web/tests/browse/views/test_origin_save.py rename to swh/web/tests/misc/test_origin_save.py index 05b508a6..2cf41bcd 100644 --- a/swh/web/tests/browse/views/test_origin_save.py +++ b/swh/web/tests/misc/test_origin_save.py @@ -1,79 +1,86 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from unittest.mock import patch from rest_framework.test import APITestCase, APIClient from swh.web.common.origin_save import ( SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED ) from swh.web.common.utils import reverse from swh.web.settings.tests import save_origin_rate_post from swh.web.tests.testcase import WebTestCase -class SwhBrowseOriginSaveTest(WebTestCase, APITestCase): +class SwhOriginSaveTest(WebTestCase, APITestCase): def setUp(self): self.client = APIClient(enforce_csrf_checks=True) self.origin = { 'type': 'git', 'url': 'https://github.com/python/cpython' } - @patch('swh.web.browse.views.origin_save.create_save_origin_request') + @patch('swh.web.misc.origin_save.create_save_origin_request') def test_save_request_form_csrf_token( self, mock_create_save_origin_request): self._mock_create_save_origin_request(mock_create_save_origin_request) - url = reverse('browse-origin-save-request', + url = reverse('origin-save-request', url_args={'origin_type': self.origin['type'], 'origin_url': self.origin['url']}) resp = self.client.post(url) self.assertEqual(resp.status_code, 403) - data = self._get_csrf_token(reverse('browse-origin-save')) + data = self._get_csrf_token(reverse('origin-save')) resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 200) - @patch('swh.web.browse.views.origin_save.create_save_origin_request') + @patch('swh.web.misc.origin_save.create_save_origin_request') def test_save_request_form_rate_limit( self, mock_create_save_origin_request): self._mock_create_save_origin_request(mock_create_save_origin_request) - url = reverse('browse-origin-save-request', + url = reverse('origin-save-request', url_args={'origin_type': self.origin['type'], 'origin_url': self.origin['url']}) - data = self._get_csrf_token(reverse('browse-origin-save')) + data = self._get_csrf_token(reverse('origin-save')) for _ in range(save_origin_rate_post): resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 200) resp = self.client.post(url, data=data) self.assertEqual(resp.status_code, 429) + def test_old_save_url_redirection(self): + url = reverse('browse-origin-save') + resp = self.client.get(url) + self.assertEqual(resp.status_code, 302) + redirect_url = reverse('origin-save') + self.assertEqual(resp['location'], redirect_url) + def _get_csrf_token(self, url): resp = self.client.get(url) return { 'csrfmiddlewaretoken': resp.cookies['csrftoken'].value } def _mock_create_save_origin_request(self, mock): expected_data = { 'origin_type': self.origin['type'], 'origin_url': self.origin['url'], 'save_request_date': datetime.now().isoformat(), 'save_request_status': SAVE_REQUEST_ACCEPTED, 'save_task_status': SAVE_TASK_NOT_YET_SCHEDULED, 'visit_date': None } mock.return_value = expected_data