diff --git a/swh/web/assets/config/.eslintrc b/swh/web/assets/config/.eslintrc index 8c1c83ab..f8dc1bfc 100644 --- a/swh/web/assets/config/.eslintrc +++ b/swh/web/assets/config/.eslintrc @@ -1,311 +1,310 @@ { "parser": "babel-eslint", "parserOptions": { "ecmaVersion": 2017, "ecmaFeatures": { "experimentalObjectRestSpread": true, "jsx": true }, "sourceType": "module", "allowImportExportEverywhere": true }, "env": { "es6": true, "node": true, "cypress/globals": true }, "plugins": [ "import", "node", "promise", "standard", "cypress" ], "globals": { "document": false, "navigator": false, "window": false, "$": false, "jQuery": false, "history": false, "localStorage": false, "sessionStorage": false, "Urls": false, "hljs": false, "Waypoint": false, "swh": false, "fetch": false, "__STATIC__": false, "Image": false, "Cookies": false, - "grecaptcha": false, "nb": false, "MathJax": false }, "rules": { "accessor-pairs": "error", "arrow-spacing": ["error", { "before": true, "after": true }], "block-spacing": ["error", "always"], "brace-style": ["error", "1tbs", { "allowSingleLine": true }], "camelcase": ["error", { "properties": "never" }], "comma-dangle": ["error", { "arrays": "never", "objects": "never", "imports": "never", "exports": "never", "functions": "never" }], "comma-spacing": ["error", { "before": false, "after": true }], "comma-style": ["error", "last"], "constructor-super": "error", "curly": ["error", "multi-line"], "dot-location": ["error", "property"], "eol-last": "error", "eqeqeq": ["error", "always", { "null": "ignore" }], "func-call-spacing": ["error", "never"], "generator-star-spacing": ["error", { "before": true, "after": true }], "handle-callback-err": ["error", "^(err|error)$"], "indent": ["error", 2, { "SwitchCase": 1, "VariableDeclarator": 1, "outerIIFEBody": 1, "MemberExpression": 1, "FunctionDeclaration": { "parameters": "first", "body": 1 }, "FunctionExpression": { "parameters": "first", "body": 1 }, "CallExpression": { "arguments": "first" }, "ArrayExpression": "first", "ObjectExpression": "first", "ImportDeclaration": "first", "flatTernaryExpressions": false, "ignoreComments": false }], "key-spacing": ["error", { "beforeColon": false, "afterColon": true }], "keyword-spacing": ["error", { "before": true, "after": true }], "new-cap": ["error", { "newIsCap": true, "capIsNew": false }], "new-parens": "error", "no-array-constructor": "error", "no-caller": "error", "no-class-assign": "error", "no-compare-neg-zero": "error", "no-cond-assign": "error", "no-const-assign": "error", "no-constant-condition": ["error", { "checkLoops": false }], "no-control-regex": "error", "no-debugger": "error", "no-delete-var": "error", "no-dupe-args": "error", "no-dupe-class-members": "error", "no-dupe-keys": "error", "no-duplicate-case": "error", "no-empty-character-class": "error", "no-empty-pattern": "error", "no-eval": "error", "no-ex-assign": "error", "no-extend-native": "error", "no-extra-bind": "error", "no-extra-boolean-cast": "error", "no-extra-parens": ["error", "functions"], "no-fallthrough": "error", "no-floating-decimal": "error", "no-func-assign": "error", "no-global-assign": "error", "no-implied-eval": "error", "no-inner-declarations": ["error", "functions"], "no-invalid-regexp": "error", "no-irregular-whitespace": "error", "no-iterator": "error", "no-label-var": "error", "no-labels": ["error", { "allowLoop": false, "allowSwitch": false }], "no-lone-blocks": "error", "no-mixed-operators": ["error", { "groups": [ ["==", "!=", "===", "!==", ">", ">=", "<", "<="], ["&&", "||"], ["in", "instanceof"] ], "allowSamePrecedence": true }], "no-mixed-spaces-and-tabs": "error", "no-multi-spaces": "error", "no-multi-str": "error", "no-multiple-empty-lines": ["error", { "max": 1, "maxEOF": 0 }], "no-negated-in-lhs": "error", "no-new": 0, "no-new-func": "error", "no-new-object": "error", "no-new-require": "error", "no-new-symbol": "error", "no-new-wrappers": "error", "no-obj-calls": "error", "no-octal": "error", "no-octal-escape": "error", "no-path-concat": "error", "no-proto": "error", "no-redeclare": "error", "no-regex-spaces": "error", "no-return-assign": ["error", "except-parens"], "no-return-await": "error", "no-self-assign": "error", "no-self-compare": "error", "no-sequences": "error", "no-shadow-restricted-names": "error", "no-sparse-arrays": "error", "no-tabs": "error", "no-template-curly-in-string": "error", "no-this-before-super": "error", "no-throw-literal": "error", "no-trailing-spaces": "error", "no-undef": "error", "no-undef-init": "error", "no-unexpected-multiline": "error", "no-unmodified-loop-condition": "error", "no-unneeded-ternary": ["error", { "defaultAssignment": false }], "no-unreachable": "error", "no-unsafe-finally": "error", "no-unsafe-negation": "error", "no-unused-expressions": ["error", { "allowShortCircuit": true, "allowTernary": true, "allowTaggedTemplates": true }], "no-unused-vars": ["error", { "vars": "all", "args": "none", "ignoreRestSiblings": true }], "no-use-before-define": ["error", { "functions": false, "classes": false, "variables": false }], "no-useless-call": "error", "no-useless-computed-key": "error", "no-useless-constructor": "error", "no-useless-escape": "error", "no-useless-rename": "error", "no-useless-return": "error", "no-whitespace-before-property": "error", "no-with": "error", "object-property-newline": ["error", { "allowMultiplePropertiesPerLine": true }], "one-var": ["error", { "initialized": "never" }], "operator-linebreak": ["error", "after", { "overrides": { "?": "before", ":": "before" } }], "padded-blocks": ["off", { "blocks": "never", "switches": "never", "classes": "never" }], "prefer-promise-reject-errors": "error", "quotes": ["error", "single", { "avoidEscape": true, "allowTemplateLiterals": true }], "rest-spread-spacing": ["error", "never"], "semi": ["error", "always"], "semi-spacing": ["error", { "before": false, "after": true }], "space-before-blocks": ["error", "always"], "space-before-function-paren": ["error", "never"], "space-in-parens": ["error", "never"], "space-infix-ops": "error", "space-unary-ops": ["error", { "words": true, "nonwords": false }], "spaced-comment": ["error", "always", { "line": { "markers": ["*package", "!", "/", ",", "="] }, "block": { "balanced": true, "markers": ["*package", "!", ",", ":", "::", "flow-include"], "exceptions": ["*"] } }], "symbol-description": "error", "template-curly-spacing": ["error", "never"], "template-tag-spacing": ["error", "never"], "unicode-bom": ["error", "never"], "use-isnan": "error", "valid-typeof": ["error", { "requireStringLiterals": true }], "wrap-iife": ["error", "any", { "functionPrototypeMethods": true }], "yield-star-spacing": ["error", "both"], "yoda": ["error", "never"], "import/export": "off", "import/first": "error", "import/no-duplicates": "error", "import/no-webpack-loader-syntax": "off", "node/no-deprecated-api": "error", "node/process-exit-as-throw": "error", "promise/param-names": "error", "standard/array-bracket-even-spacing": ["error", "either"], "standard/computed-property-even-spacing": ["error", "even"], "standard/no-callback-literal": "error", "standard/object-curly-even-spacing": ["error", "either"] } } \ No newline at end of file diff --git a/swh/web/assets/src/bundles/browse/origin-save.js b/swh/web/assets/src/bundles/browse/origin-save.js index be9a2ca2..871038d1 100644 --- a/swh/web/assets/src/bundles/browse/origin-save.js +++ b/swh/web/assets/src/bundles/browse/origin-save.js @@ -1,276 +1,265 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import {handleFetchError, csrfPost, isGitRepoUrl, removeUrlFragment} from 'utils/functions'; import {validate} from 'validate.js'; let saveRequestsTable; function originSaveRequest(originType, originUrl, acceptedCallback, pendingCallback, errorCallback) { let addSaveOriginRequestUrl = Urls.browse_origin_save_request(originType, originUrl); - let grecaptchaData = {}; - if (swh.webapp.isReCaptchaActivated()) { - grecaptchaData['g-recaptcha-response'] = grecaptcha.getResponse(); - } let headers = { 'Accept': 'application/json', 'Content-Type': 'application/json' }; - let body = JSON.stringify(grecaptchaData); $('.swh-processing-save-request').css('display', 'block'); - csrfPost(addSaveOriginRequestUrl, headers, body) + csrfPost(addSaveOriginRequestUrl, headers) .then(handleFetchError) .then(response => response.json()) .then(data => { $('.swh-processing-save-request').css('display', 'none'); if (data.save_request_status === 'accepted') { acceptedCallback(); } else { pendingCallback(); } - if (swh.webapp.isReCaptchaActivated()) { - grecaptcha.reset(); - } }) .catch(response => { $('.swh-processing-save-request').css('display', 'none'); if (response.status === 403) { errorCallback(); } - if (swh.webapp.isReCaptchaActivated()) { - grecaptcha.reset(); - } }); } export function initOriginSave() { $(document).ready(() => { $.fn.dataTable.ext.errMode = 'none'; fetch(Urls.browse_origin_save_types_list()) .then(response => response.json()) .then(data => { for (let originType of data) { $('#swh-input-origin-type').append(``); } }); saveRequestsTable = $('#swh-origin-save-requests') .on('error.dt', (e, settings, techNote, message) => { $('#swh-origin-save-request-list-error').text('An error occurred while retrieving the save requests list'); console.log(message); }) .DataTable({ serverSide: true, ajax: Urls.browse_origin_save_requests_list('all'), searchDelay: 1000, columns: [ { data: 'save_request_date', name: 'request_date', render: (data, type, row) => { if (type === 'display') { let date = new Date(data); return date.toLocaleString(); } return data; } }, { data: 'origin_type', name: 'origin_type' }, { data: 'origin_url', name: 'origin_url', render: (data, type, row) => { if (type === 'display') { const sanitizedURL = $.fn.dataTable.render.text().display(data); return `${sanitizedURL}`; } return data; } }, { data: 'save_request_status', name: 'status' }, { data: 'save_task_status', name: 'loading_task_status', render: (data, type, row) => { if (data === 'succeed') { let browseOriginUrl = Urls.browse_origin(row.origin_url); if (row.visit_date) { browseOriginUrl += `visit/${row.visit_date}/`; } return `${data}`; } return data; } } ], scrollY: '50vh', scrollCollapse: true, order: [[0, 'desc']], responsive: { details: { type: 'none' } } }); $('#swh-origin-save-requests-list-tab').on('shown.bs.tab', () => { saveRequestsTable.draw(); window.location.hash = '#requests'; }); $('#swh-origin-save-request-create-tab').on('shown.bs.tab', () => { removeUrlFragment(); }); let saveRequestAcceptedAlert = ``; let saveRequestPendingAlert = ``; let saveRequestRejectedAlert = ``; $('#swh-save-origin-form').submit(event => { event.preventDefault(); event.stopPropagation(); $('.alert').alert('close'); if (event.target.checkValidity()) { $(event.target).removeClass('was-validated'); let originType = $('#swh-input-origin-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), () => { $('#swh-origin-save-request-status').css('color', 'red'); $('#swh-origin-save-request-status').html(saveRequestRejectedAlert); }); } else { $(event.target).addClass('was-validated'); } }); $('#swh-show-origin-save-requests-list').on('click', (event) => { event.preventDefault(); $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); }); $('#swh-input-origin-url').on('input', function(event) { let originUrl = $(this).val().trim(); $(this).val(originUrl); $('#swh-input-origin-type option').each(function() { let val = $(this).val(); if (val && originUrl.includes(val)) { $(this).prop('selected', true); } }); }); if (window.location.hash === '#requests') { $('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show'); } }); } export function validateSaveOriginUrl(input) { let originUrl = input.value.trim(); let validUrl = validate({website: originUrl}, { website: { url: { schemes: ['http', 'https', 'svn', 'git'] } } }) === undefined; let originType = $('#swh-input-origin-type').val(); if (originType === 'git' && validUrl) { // additional checks for well known code hosting providers let githubIdx = originUrl.indexOf('://github.com'); let gitlabIdx = originUrl.indexOf('://gitlab.'); let gitSfIdx = originUrl.indexOf('://git.code.sf.net'); let bitbucketIdx = originUrl.indexOf('://bitbucket.org'); if (githubIdx !== -1 && githubIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'github.com'); } else if (gitlabIdx !== -1 && gitlabIdx <= 5) { let startIdx = gitlabIdx + 3; let idx = originUrl.indexOf('/', startIdx); if (idx !== -1) { let gitlabDomain = originUrl.substr(startIdx, idx - startIdx); // GitLab repo url needs to be suffixed by '.git' in order to be successfully loaded // This is due to a bug in dulwich < 0.19.11. // TODO: remove this check once dulwich >= 0.19.11 is used in production validUrl = isGitRepoUrl(originUrl, gitlabDomain) && originUrl.endsWith('.git'); } else { validUrl = false; } } else if (gitSfIdx !== -1 && gitSfIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'git.code.sf.net/p'); } else if (bitbucketIdx !== -1 && bitbucketIdx <= 5) { validUrl = isGitRepoUrl(originUrl, 'bitbucket.org'); } } if (validUrl) { input.setCustomValidity(''); } else { input.setCustomValidity('The origin url is not valid or does not reference a code repository'); } } export function initTakeNewSnapshot() { let newSnapshotRequestAcceptedAlert = ``; let newSnapshotRequestPendingAlert = ``; let newSnapshotRequestRejectedAlert = ``; $(document).ready(() => { $('#swh-take-new-snapshot-form').submit(event => { event.preventDefault(); event.stopPropagation(); let originType = $('#swh-input-origin-type').val(); let originUrl = $('#swh-input-origin-url').val(); originSaveRequest(originType, originUrl, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), () => { $('#swh-take-new-snapshot-request-status').css('color', 'red'); $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRejectedAlert); }); }); }); } diff --git a/swh/web/assets/src/bundles/webapp/webapp-utils.js b/swh/web/assets/src/bundles/webapp/webapp-utils.js index 340d9d3b..f40d27d0 100644 --- a/swh/web/assets/src/bundles/webapp/webapp-utils.js +++ b/swh/web/assets/src/bundles/webapp/webapp-utils.js @@ -1,213 +1,203 @@ import objectFitImages from 'object-fit-images'; import {Layout} from 'admin-lte'; import {selectText} from 'utils/functions'; import {BREAKPOINT_MD} from 'utils/constants'; let collapseSidebar = false; let previousSidebarState = localStorage.getItem('swh-sidebar-collapsed'); if (previousSidebarState !== undefined) { collapseSidebar = JSON.parse(previousSidebarState); } // adapt implementation of fixLayoutHeight from admin-lte Layout.prototype.fixLayoutHeight = () => { let heights = { window: $(window).height(), header: $('.main-header').outerHeight(), footer: $('.footer').outerHeight(), sidebar: $('.main-sidebar').height(), topbar: $('.swh-top-bar').height() }; let offset = 10; $('.content-wrapper').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); $('.main-sidebar').css('min-height', heights.window - heights.topbar - heights.header - heights.footer - offset); }; $(document).on('DOMContentLoaded', () => { // set state to collapsed on smaller devices if ($(window).width() < BREAKPOINT_MD) { collapseSidebar = true; } // restore previous sidebar state (collapsed/expanded) if (collapseSidebar) { // hack to avoid animated transition for collapsing sidebar // when loading a page let sidebarTransition = $('.main-sidebar, .main-sidebar:before').css('transition'); let sidebarEltsTransition = $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition'); $('.main-sidebar, .main-sidebar:before').css('transition', 'none'); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', 'none'); $('body').addClass('sidebar-collapse'); $('.swh-words-logo-swh').css('visibility', 'visible'); // restore transitions for user navigation setTimeout(() => { $('.main-sidebar, .main-sidebar:before').css('transition', sidebarTransition); $('.sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info').css('transition', sidebarEltsTransition); }); } }); $(document).on('collapsed.lte.pushmenu', event => { if ($('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); $(document).on('shown.lte.pushmenu', event => { $('.swh-words-logo-swh').css('visibility', 'hidden'); }); function ensureNoFooterOverflow() { $('body').css('padding-bottom', $('footer').outerHeight() + 'px'); } $(document).ready(() => { // redirect to last browse page if any when clicking on the 'Browse' entry // in the sidebar $(`.swh-browse-link`).click(event => { let lastBrowsePage = sessionStorage.getItem('last-browse-page'); if (lastBrowsePage) { event.preventDefault(); window.location = lastBrowsePage; } }); // ensure footer do not overflow main content for mobile devices // or after resizing the browser window ensureNoFooterOverflow(); $(window).resize(function() { ensureNoFooterOverflow(); if ($('body').hasClass('sidebar-collapse') && $('body').width() >= BREAKPOINT_MD) { $('.swh-words-logo-swh').css('visibility', 'visible'); } }); // activate css polyfill 'object-fit: contain' in old browsers objectFitImages(); // reparent the modals to the top navigation div in order to be able // to display them $('.swh-browse-top-navigation').append($('.modal')); let selectedCode = null; function getCodeOrPreEltUnderPointer(e) { let elts = document.elementsFromPoint(e.clientX, e.clientY); for (let elt of elts) { if (elt.nodeName === 'CODE' || elt.nodeName === 'PRE') { return elt; } } return null; } // click handler to set focus on code block for copy $(document).click(e => { selectedCode = getCodeOrPreEltUnderPointer(e); }); function selectCode(event, selectedCode) { if (selectedCode) { let hljsLnCodeElts = $(selectedCode).find('.hljs-ln-code'); if (hljsLnCodeElts.length) { selectText(hljsLnCodeElts[0], hljsLnCodeElts[hljsLnCodeElts.length - 1]); } else { selectText(selectedCode.firstChild, selectedCode.lastChild); } event.preventDefault(); } } // select the whole text of focused code block when user // double clicks or hits Ctrl+A $(document).dblclick(e => { if ((e.ctrlKey || e.metaKey)) { selectCode(e, getCodeOrPreEltUnderPointer(e)); } }); $(document).keydown(e => { if ((e.ctrlKey || e.metaKey) && e.key === 'a') { selectCode(e, selectedCode); } }); // show/hide back-to-top button let scrollThreshold = 0; scrollThreshold += $('.swh-top-bar').height() || 0; scrollThreshold += $('.navbar').height() || 0; $(window).scroll(() => { if ($(window).scrollTop() > scrollThreshold) { $('#back-to-top').css('display', 'block'); } else { $('#back-to-top').css('display', 'none'); } }); }); export function initPage(page) { $(document).ready(() => { // set relevant sidebar link to page active $(`.swh-${page}-item`).addClass('active'); $(`.swh-${page}-link`).addClass('active'); // triggered when unloading the current page $(window).on('unload', () => { // backup sidebar state (collapsed/expanded) let sidebarCollapsed = $('body').hasClass('sidebar-collapse'); localStorage.setItem('swh-sidebar-collapsed', JSON.stringify(sidebarCollapsed)); // backup current browse page if (page === 'browse') { sessionStorage.setItem('last-browse-page', window.location); } }); }); } export function showModalMessage(title, message) { $('#swh-web-modal-message .modal-title').text(title); $('#swh-web-modal-message .modal-content p').text(message); $('#swh-web-modal-message').modal('show'); } export function showModalConfirm(title, message, callback) { $('#swh-web-modal-confirm .modal-title').text(title); $('#swh-web-modal-confirm .modal-content p').text(message); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').bind('click', () => { callback(); $('#swh-web-modal-confirm').modal('hide'); $('#swh-web-modal-confirm #swh-web-modal-confirm-ok-btn').unbind('click'); }); $('#swh-web-modal-confirm').modal('show'); } let swhObjectIcons; export function setSwhObjectIcons(icons) { swhObjectIcons = icons; } export function getSwhObjectIcon(swhObjectType) { return swhObjectIcons[swhObjectType]; } -let reCaptchaActivated; - -export function setReCaptchaActivated(activated) { - reCaptchaActivated = activated; -} - -export function isReCaptchaActivated() { - return reCaptchaActivated; -} - let browsedSwhObjectMetadata = {}; export function setBrowsedSwhObjectMetadata(metadata) { browsedSwhObjectMetadata = metadata; } export function getBrowsedSwhObjectMetadata() { return browsedSwhObjectMetadata; } diff --git a/swh/web/browse/views/origin_save.py b/swh/web/browse/views/origin_save.py index 14b77944..bdbc0ae3 100644 --- a/swh/web/browse/views/origin_save.py +++ b/swh/web/browse/views/origin_save.py @@ -1,86 +1,90 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json from django.core.paginator import Paginator from django.http import HttpResponse, HttpResponseForbidden -from django.views.decorators.http import require_POST + +from rest_framework.decorators import api_view, authentication_classes from swh.web.browse.browseurls import browse_route from swh.web.common.exc import ForbiddenExc from swh.web.common.models import SaveOriginRequest -from swh.web.common.utils import is_recaptcha_valid from swh.web.common.origin_save import ( create_save_origin_request, get_savable_origin_types, get_save_origin_requests_from_queryset ) +from swh.web.common.throttling import throttle_scope +from swh.web.common.utils import EnforceCSRFAuthentication @browse_route(r'origin/save/(?P.+)/url/(?P.+)/', view_name='browse-origin-save-request') -@require_POST +@api_view(['POST']) +@authentication_classes((EnforceCSRFAuthentication, )) +@throttle_scope('swh_save_origin') def _browse_origin_save_request(request, origin_type, origin_url): - body_unicode = request.body.decode('utf-8') - body = json.loads(body_unicode) - if is_recaptcha_valid(request, body.get('g-recaptcha-response')): - try: - response = json.dumps(create_save_origin_request(origin_type, - origin_url), - separators=(',', ': ')) - return HttpResponse(response, content_type='application/json') - except ForbiddenExc as exc: - return HttpResponseForbidden(str(exc)) - else: - return HttpResponseForbidden('The reCAPTCHA could not be validated !') + """ + This view is called through AJAX from the save code now form of swh-web. + We use DRF here as we want to rate limit the number of submitted requests + per user to avoid being possibly flooded by bots. + """ + try: + response = json.dumps(create_save_origin_request(origin_type, + origin_url), + separators=(',', ': ')) + return HttpResponse(response, content_type='application/json') + except ForbiddenExc as exc: + return HttpResponseForbidden(str(exc)) @browse_route(r'origin/save/types/list/', view_name='browse-origin-save-types-list') def _browse_origin_save_types_list(request): origin_types = json.dumps(get_savable_origin_types(), separators=(',', ': ')) return HttpResponse(origin_types, content_type='application/json') @browse_route(r'origin/save/requests/list/(?P.+)/', view_name='browse-origin-save-requests-list') def _browse_origin_save_requests_list(request, status): if status != 'all': save_requests = SaveOriginRequest.objects.filter(status=status) else: save_requests = SaveOriginRequest.objects.all() table_data = {} table_data['recordsTotal'] = save_requests.count() table_data['draw'] = int(request.GET['draw']) search_value = request.GET['search[value]'] column_order = request.GET['order[0][column]'] field_order = request.GET['columns[%s][name]' % column_order] order_dir = request.GET['order[0][dir]'] if order_dir == 'desc': field_order = '-' + field_order save_requests = save_requests.order_by(field_order) length = int(request.GET['length']) page = int(request.GET['start']) / length + 1 save_requests = get_save_origin_requests_from_queryset(save_requests) if search_value: save_requests = \ [sr for sr in save_requests if search_value.lower() in sr['save_request_status'].lower() or search_value.lower() in sr['save_task_status'].lower() or search_value.lower() in sr['origin_type'].lower() or search_value.lower() in sr['origin_url'].lower()] table_data['recordsFiltered'] = len(save_requests) paginator = Paginator(save_requests, length) table_data['data'] = paginator.page(page).object_list table_data_json = json.dumps(table_data, separators=(',', ': ')) return HttpResponse(table_data_json, content_type='application/json') diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py index 456d04b7..71926c26 100644 --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -1,355 +1,336 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import docutils.parsers.rst import docutils.utils import re -import requests from datetime import datetime, timezone from dateutil import parser as date_parser from dateutil import tz from django.urls import reverse as django_reverse from django.http import QueryDict +from rest_framework.authentication import SessionAuthentication + from swh.model.exceptions import ValidationError from swh.model.identifiers import ( persistent_identifier, parse_persistent_identifier, CONTENT, DIRECTORY, RELEASE, REVISION, SNAPSHOT ) from swh.web.common.exc import BadInputExc -from swh.web.config import get_config swh_object_icons = { 'branch': 'fa fa-code-fork', 'branches': 'fa fa-code-fork', 'content': 'fa fa-file-text', 'directory': 'fa fa-folder', 'person': 'fa fa-user', 'revisions history': 'fa fa-history', 'release': 'fa fa-tag', 'releases': 'fa fa-tag', 'revision': 'octicon-git-commit', 'snapshot': 'fa fa-camera', 'visits': 'fa fa-calendar', } def reverse(viewname, url_args=None, query_params=None, current_app=None, urlconf=None): """An override of django reverse function supporting query parameters. Args: viewname (str): the name of the django view from which to compute a url url_args (dict): dictionary of url arguments indexed by their names query_params (dict): dictionary of query parameters to append to the reversed url current_app (str): the name of the django app tighten to the view urlconf (str): url configuration module Returns: str: the url of the requested view with processed arguments and query parameters """ if url_args: url_args = {k: v for k, v in url_args.items() if v is not None} url = django_reverse(viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app) if query_params: query_params = {k: v for k, v in query_params.items() if v} if query_params and len(query_params) > 0: query_dict = QueryDict('', mutable=True) for k in sorted(query_params.keys()): query_dict[k] = query_params[k] url += ('?' + query_dict.urlencode(safe='/;:')) return url def datetime_to_utc(date): """Returns datetime in UTC without timezone info Args: date (datetime.datetime): input datetime with timezone info Returns: datetime.datetime: datetime in UTC without timezone info """ if date.tzinfo: return date.astimezone(tz.gettz('UTC')).replace(tzinfo=timezone.utc) else: return date def parse_timestamp(timestamp): """Given a time or timestamp (as string), parse the result as UTC datetime. Returns: datetime.datetime: a timezone-aware datetime representing the parsed value or None if the parsing fails. Samples: - 2016-01-12 - 2016-01-12T09:19:12+0100 - Today is January 1, 2047 at 8:21:00AM - 1452591542 """ if not timestamp: return None try: date = date_parser.parse(timestamp, ignoretz=False, fuzzy=True) return datetime_to_utc(date) except Exception: try: return datetime.utcfromtimestamp(float(timestamp)).replace( tzinfo=timezone.utc) except (ValueError, OverflowError) as e: raise BadInputExc(e) def shorten_path(path): """Shorten the given path: for each hash present, only return the first 8 characters followed by an ellipsis""" sha256_re = r'([0-9a-f]{8})[0-9a-z]{56}' sha1_re = r'([0-9a-f]{8})[0-9a-f]{32}' ret = re.sub(sha256_re, r'\1...', path) return re.sub(sha1_re, r'\1...', ret) def format_utc_iso_date(iso_date, fmt='%d %B %Y, %H:%M UTC'): """Turns a string representation of an ISO 8601 date string to UTC and format it into a more human readable one. For instance, from the following input string: '2017-05-04T13:27:13+02:00' the following one is returned: '04 May 2017, 11:27 UTC'. Custom format string may also be provided as parameter Args: iso_date (str): a string representation of an ISO 8601 date fmt (str): optional date formatting string Returns: str: a formatted string representation of the input iso date """ if not iso_date: return iso_date date = parse_timestamp(iso_date) return date.strftime(fmt) def gen_path_info(path): """Function to generate path data navigation for use with a breadcrumb in the swh web ui. For instance, from a path /folder1/folder2/folder3, it returns the following list:: [{'name': 'folder1', 'path': 'folder1'}, {'name': 'folder2', 'path': 'folder1/folder2'}, {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] Args: path: a filesystem path Returns: list: a list of path data for navigation as illustrated above. """ path_info = [] if path: sub_paths = path.strip('/').split('/') path_from_root = '' for p in sub_paths: path_from_root += '/' + p path_info.append({'name': p, 'path': path_from_root.strip('/')}) return path_info def get_swh_persistent_id(object_type, object_id, scheme_version=1): """ Returns the persistent identifier for a swh object based on: * the object type * the object id * the swh identifiers scheme version Args: object_type (str): the swh object type (content/directory/release/revision/snapshot) object_id (str): the swh object id (hexadecimal representation of its hash value) scheme_version (int): the scheme version of the swh persistent identifiers Returns: str: the swh object persistent identifier Raises: BadInputExc: if the provided parameters do not enable to generate a valid identifier """ try: swh_id = persistent_identifier(object_type, object_id, scheme_version) except ValidationError as e: raise BadInputExc('Invalid object (%s) for swh persistent id. %s' % (object_id, e)) else: return swh_id def resolve_swh_persistent_id(swh_id, query_params=None): """ Try to resolve a Software Heritage persistent id into an url for browsing the pointed object. Args: swh_id (str): a Software Heritage persistent identifier query_params (django.http.QueryDict): optional dict filled with query parameters to append to the browse url Returns: dict: a dict with the following keys: * **swh_id_parsed (swh.model.identifiers.PersistentId)**: the parsed identifier * **browse_url (str)**: the url for browsing the pointed object Raises: BadInputExc: if the provided identifier can not be parsed """ # noqa try: swh_id_parsed = parse_persistent_identifier(swh_id) object_type = swh_id_parsed.object_type object_id = swh_id_parsed.object_id browse_url = None query_dict = QueryDict('', mutable=True) if query_params and len(query_params) > 0: for k in sorted(query_params.keys()): query_dict[k] = query_params[k] if 'origin' in swh_id_parsed.metadata: query_dict['origin'] = swh_id_parsed.metadata['origin'] if object_type == CONTENT: query_string = 'sha1_git:' + object_id fragment = '' if 'lines' in swh_id_parsed.metadata: lines = swh_id_parsed.metadata['lines'].split('-') fragment += '#L' + lines[0] if len(lines) > 1: fragment += '-L' + lines[1] browse_url = reverse('browse-content', url_args={'query_string': query_string}, query_params=query_dict) + fragment elif object_type == DIRECTORY: browse_url = reverse('browse-directory', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == RELEASE: browse_url = reverse('browse-release', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == REVISION: browse_url = reverse('browse-revision', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == SNAPSHOT: browse_url = reverse('browse-snapshot', url_args={'snapshot_id': object_id}, query_params=query_dict) except ValidationError as ve: raise BadInputExc('Error when parsing identifier. %s' % ' '.join(ve.messages)) else: return {'swh_id_parsed': swh_id_parsed, 'browse_url': browse_url} def parse_rst(text, report_level=2): """ Parse a reStructuredText string with docutils. Args: text (str): string with reStructuredText markups in it report_level (int): level of docutils report messages to print (1 info 2 warning 3 error 4 severe 5 none) Returns: docutils.nodes.document: a parsed docutils document """ parser = docutils.parsers.rst.Parser() components = (docutils.parsers.rst.Parser,) settings = docutils.frontend.OptionParser( components=components).get_default_values() settings.report_level = report_level document = docutils.utils.new_document('rst-doc', settings=settings) parser.parse(text, document) return document def get_client_ip(request): """ Return the client IP address from an incoming HTTP request. Args: request (django.http.HttpRequest): the incoming HTTP request Returns: str: The client IP address """ x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') if x_forwarded_for: ip = x_forwarded_for.split(',')[0] else: ip = request.META.get('REMOTE_ADDR') return ip -def is_recaptcha_valid(request, recaptcha_response): +def context_processor(request): """ - Verify if the response for Google reCAPTCHA is valid. - - Args: - request (django.http.HttpRequest): the incoming HTTP request - recaptcha_response (str): the reCAPTCHA response - - Returns: - bool: Whether the reCAPTCHA response is valid or not + Django context processor used to inject variables + in all swh-web templates. """ - config = get_config() - if config['grecaptcha']['activated'] is False: - recaptcha_valid = True - else: - recaptcha_valid = requests.post( - config['grecaptcha']['validation_url'], - data={ - 'secret': config['grecaptcha']['private_key'], - 'response': recaptcha_response, - 'remoteip': get_client_ip(request) - }, - verify=True - ).json().get("success", False) - return recaptcha_valid + return {'swh_object_icons': swh_object_icons} -def context_processor(request): +class EnforceCSRFAuthentication(SessionAuthentication): """ - Django context processor used to inject variables - in all swh-web templates. + Helper class to enforce CSRF validation on a DRF view + when a user is not authenticated. """ - config = get_config() - return {'swh_object_icons': swh_object_icons, - 'grecaptcha_activated': config['grecaptcha']['activated'], - 'grecaptcha_site_key': config['grecaptcha']['site_key']} + def authenticate(self, request): + user = getattr(request._request, 'user', None) + self.enforce_csrf(request) + return (user, None) diff --git a/swh/web/config.py b/swh/web/config.py index 9e7dc805..ea1d462f 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,154 +1,148 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from swh.core import config from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { 'allowed_hosts': ('list', []), 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', 'timeout': 10, }, }), 'indexer_storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5007/', 'timeout': 1, } }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'serve_assets': ('bool', False), 'host': ('string', '127.0.0.1'), 'port': ('int', 5004), 'secret_key': ('string', 'development key'), # do not display code highlighting for content > 1MB 'content_display_max_size': ('int', 5 * 1024 * 1024), 'snapshot_content_max_size': ('int', 1000), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '120/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '10/h' }, 'exempted_networks': ['127.0.0.0/8'] } } }), 'vault': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5005/', } }), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5008/' } }), - 'grecaptcha': ('dict', { - 'activated': True, - 'validation_url': 'https://www.google.com/recaptcha/api/siteverify', - 'site_key': '', - 'private_key': '' - }), 'development_db': ('string', os.path.join(SETTINGS_DIR, 'db.sqlite3')), 'production_db': ('string', '/var/lib/swh/web.sqlite3'), 'deposit': ('dict', { 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', 'private_api_user': 'swhworker', 'private_api_password': '' }), 'coverage_count_origins': ('bool', False) } swhweb_config = {} def get_config(config_file='web/web'): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get('SWH_CONFIG_FILENAME') if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) swhweb_config['vault'] = get_vault(**swhweb_config['vault']) swhweb_config['indexer_storage'] = \ get_indexer_storage(**swhweb_config['indexer_storage']) swhweb_config['scheduler'] = get_scheduler( **swhweb_config['scheduler']) return swhweb_config def storage(): """Return the current application's storage. """ return get_config()['storage'] def vault(): """Return the current application's vault. """ return get_config()['vault'] def indexer_storage(): """Return the current application's indexer storage. """ return get_config()['indexer_storage'] def scheduler(): """Return the current application's scheduler. """ return get_config()['scheduler'] diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py index ea137acb..69b7cbe8 100644 --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -1,93 +1,88 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django tests settings for swh-web. """ import sys from swh.web.config import get_config -from swh.web.tests.data import get_tests_data, override_storages scope1_limiter_rate = 3 scope1_limiter_rate_post = 1 scope2_limiter_rate = 5 scope2_limiter_rate_post = 2 scope3_limiter_rate = 1 scope3_limiter_rate_post = 1 +save_origin_rate_post = 10 swh_web_config = get_config() swh_web_config.update({ 'debug': False, 'secret_key': 'test', 'throttling': { 'cache_uri': None, 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '60/min' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', - 'POST': '10/h' - }, - 'exempted_networks': ['127.0.0.0/8'] + 'POST': '%s/h' % save_origin_rate_post, + } }, 'scope1': { 'limiter_rate': { 'default': '%s/min' % scope1_limiter_rate, 'POST': '%s/min' % scope1_limiter_rate_post, } }, 'scope2': { 'limiter_rate': { 'default': '%s/min' % scope2_limiter_rate, 'POST': '%s/min' % scope2_limiter_rate_post } }, 'scope3': { 'limiter_rate': { 'default': '%s/min' % scope3_limiter_rate, 'POST': '%s/min' % scope3_limiter_rate_post }, 'exempted_networks': ['127.0.0.0/8'] } } } }) from .common import * # noqa from .common import ALLOWED_HOSTS, LOGGING # noqa # when not running unit tests, make the webapp fetch data from memory storages if 'pytest' not in sys.argv[0]: swh_web_config.update({ 'debug': True, - 'grecaptcha': { - 'activated': False, - 'site_key': '', - 'private_key': '' - } }) + from swh.web.tests.data import get_tests_data, override_storages # noqa test_data = get_tests_data() override_storages(test_data['storage'], test_data['idx_storage']) else: ALLOWED_HOSTS += ['testserver'] # Silent DEBUG output when running unit tests LOGGING['handlers']['console']['level'] = 'INFO' diff --git a/swh/web/templates/browse/origin-save.html b/swh/web/templates/browse/origin-save.html index 13b2e55f..4f0d3b52 100644 --- a/swh/web/templates/browse/origin-save.html +++ b/swh/web/templates/browse/origin-save.html @@ -1,135 +1,113 @@ {% extends "./layout.html" %} {% comment %} Copyright (C) 2018-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} -{% block header %} -{{ block.super }} -{% if grecaptcha_activated %} - -{% endif %} -{% endblock %} - {% block navbar-content %}

Save code now

{% endblock %} {% block browse-content %}

You can contribute to extend the content of the Software Heritage archive by submitting an origin save request. To do so, fill the required info in the form below:

  • Origin type: the type of version control system the software origin is using. Currently, the supported types are:
  • Origin url: the url of the remote repository for the software origin.
    In order to avoid saving errors from Software Heritage, you should provide the clone/checkout url as given by the provider hosting the software origin.
    It can easily be found in the web interface used to browse the software origin.
    For instance, if you want to save a git origin into the archive, you should check that the command $ git clone <origin_url>
    does not return an error before submitting a request.

Once submitted, your save request can either be:

  • accepted: a visit to the provided origin will then be scheduled by Software Heritage in order to load its content into the archive as soon as possible
  • rejected: the provided origin url is blacklisted and no visit will be scheduled
  • put in pending state: a manual review will then be performed in order to determine if the origin can be safely loaded or not into the archive

Once a save request has been accepted, you can follow its current status in the submitted save requests list.

{% csrf_token %}
The origin type must be specified
The origin url is not valid or does not reference a code repository
- {% if not grecaptcha_activated %} -
- - -
- {% endif %} -
-
- {% if grecaptcha_activated %} -
-
-
-
-
-
-
+
- {% endif %} +
Date Type Url Request Status

{% endblock %} \ No newline at end of file diff --git a/swh/web/templates/includes/take-new-snapshot.html b/swh/web/templates/includes/take-new-snapshot.html index 7fd6b313..376f1ece 100644 --- a/swh/web/templates/includes/take-new-snapshot.html +++ b/swh/web/templates/includes/take-new-snapshot.html @@ -1,92 +1,76 @@ {% comment %} Copyright (C) 2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load static %} {% load swh_templatetags %} {% if snapshot_context and snapshot_context.origin_info and snapshot_context.origin_info.type|origin_type_savable %} - {% if grecaptcha_activated %} - - {% endif %} -
{% endif %} \ No newline at end of file diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index 2746a525..0820896d 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,214 +1,213 @@ {% comment %} Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block title %}{% endblock %} {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% block header %}{% endblock %}
{% block content %}{% endblock %}
{% include "includes/global-modals.html" %}
back to top
diff --git a/swh/web/tests/browse/views/test_origin_save.py b/swh/web/tests/browse/views/test_origin_save.py index 944cf701..05b508a6 100644 --- a/swh/web/tests/browse/views/test_origin_save.py +++ b/swh/web/tests/browse/views/test_origin_save.py @@ -1,143 +1,79 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import json from datetime import datetime -from hypothesis import given from unittest.mock import patch +from rest_framework.test import APITestCase, APIClient + from swh.web.common.origin_save import ( SAVE_REQUEST_ACCEPTED, SAVE_TASK_NOT_YET_SCHEDULED ) -from swh.web.config import get_config from swh.web.common.utils import reverse -from swh.web.tests.strategies import origin +from swh.web.settings.tests import save_origin_rate_post from swh.web.tests.testcase import WebTestCase -class SwhBrowseOriginSaveTest(WebTestCase): - - @given(origin()) - def test_recaptcha_activation_in_gui(self, origin): - - swh_web_config = get_config() - - for captcha_activated in (True, False): +class SwhBrowseOriginSaveTest(WebTestCase, APITestCase): - swh_web_config.update({ - 'grecaptcha': { - 'activated': captcha_activated, - 'site_key': '' - } - }) - - url = reverse('browse-origin-save') - resp = self.client.get(url) + def setUp(self): + self.client = APIClient(enforce_csrf_checks=True) + self.origin = { + 'type': 'git', + 'url': 'https://github.com/python/cpython' + } - captcha_script_url = 'https://www.google.com/recaptcha/api.js' - captcha_dom_elt = '