diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js
index b7ab39b6..556cf769 100644
--- a/cypress/integration/origin-save.spec.js
+++ b/cypress/integration/origin-save.spec.js
@@ -1,234 +1,251 @@
/**
* Copyright (C) 2019-2020 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
let url;
let origin;
const $ = Cypress.$;
const saveCodeMsg = {
'success': 'The "save code now" request has been accepted and will be processed as soon as possible.',
'warning': 'The "save code now" request has been put in pending state and may be accepted for processing after manual review.',
'rejected': 'The "save code now" request has been rejected because the provided origin url is blacklisted.',
'rateLimit': 'The rate limit for "save code now" requests has been reached. Please try again later.',
'unknownError': 'An unexpected error happened when submitting the "save code now request',
'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.'
};
function makeOriginSaveRequest(originType, originUrl) {
cy.get('#swh-input-visit-type')
.select(originType)
.get('#swh-input-origin-url')
.type(originUrl)
.get('#swh-save-origin-form')
.submit();
}
function checkAlertVisible(alertType, msg) {
cy.get('#swh-origin-save-request-status')
.should('be.visible')
.find(`.alert-${alertType}`)
.should('be.visible')
.and('contain', msg);
}
// Stub requests to save an origin
function stubSaveRequest(requestUrl, objectType, status, originUrl, taskStatus,
responseStatus = 200, errorMessage = '') {
let response;
if (responseStatus !== 200 && errorMessage) {
response = {'detail': errorMessage};
} else {
response = genOriginSaveResponse(objectType, status, originUrl, Date().toString(), taskStatus);
}
cy.route({
method: 'POST',
status: responseStatus,
url: requestUrl,
response: response
}).as('saveRequest');
}
// Mocks API response : /save/(:object_type)/(:origin_url)
// object_type : {'git', 'hg', 'svn'}
function genOriginSaveResponse(objectType, saveRequestStatus, originUrl, saveRequestDate, saveTaskStatus) {
return {
'visit_type': objectType,
'save_request_status': saveRequestStatus,
'origin_url': originUrl,
'id': 1,
'save_request_date': saveRequestDate,
'save_task_status': saveTaskStatus,
'visit_date': null
};
};
describe('Origin Save Tests', function() {
before(function() {
url = this.Urls.origin_save();
origin = this.origin[0];
this.originSaveUrl = this.Urls.origin_save_request(origin.type, origin.url);
});
beforeEach(function() {
cy.visit(url);
cy.server();
});
it('should display accepted message when accepted', function() {
stubSaveRequest(this.originSaveUrl, origin.type, 'accepted',
origin.url, 'not yet scheduled');
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('success', saveCodeMsg['success']);
});
});
it('should validate gitlab subproject url', function() {
const gitlabSubProjectUrl = 'https://gitlab.com/user/project/sub/';
const originSaveUrl = this.Urls.origin_save_request('git', gitlabSubProjectUrl);
stubSaveRequest(originSaveUrl, 'git', 'accepted',
gitlabSubProjectUrl, 'not yet scheduled');
makeOriginSaveRequest('git', gitlabSubProjectUrl);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('success', saveCodeMsg['success']);
});
});
it('should display warning message when pending', function() {
stubSaveRequest(this.originSaveUrl, origin.type, 'pending',
origin.url, 'not created');
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('warning', saveCodeMsg['warning']);
});
});
it('should show error when csrf validation failed (status: 403)', function() {
stubSaveRequest(this.originSaveUrl, origin.type, 'rejected',
origin.url, 'not created', 403, saveCodeMsg['csrfError']);
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('danger', saveCodeMsg['csrfError']);
});
});
it('should show error when origin is rejected (status: 403)', function() {
stubSaveRequest(this.originSaveUrl, origin.type, 'rejected',
origin.url, 'not created', 403, saveCodeMsg['rejected']);
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('danger', saveCodeMsg['rejected']);
});
});
it('should show error when rate limited (status: 429)', function() {
stubSaveRequest(this.originSaveUrl, origin.type,
'Request was throttled. Expected available in 60 seconds.',
origin.url, 'not created', 429);
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('danger', saveCodeMsg['rateLimit']);
});
});
it('should show error when unknown error occurs (status other than 200, 403, 429)', function() {
stubSaveRequest(this.originSaveUrl, origin.type, 'Error',
origin.url, 'not created', 406);
makeOriginSaveRequest(origin.type, origin.url);
cy.wait('@saveRequest').then(() => {
checkAlertVisible('danger', saveCodeMsg['unknownError']);
});
});
it('should display origin save info in the requests table', function() {
cy.fixture('origin-save').as('originSaveJSON');
cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
cy.get('#swh-origin-save-requests-list-tab').click();
cy.get('tbody tr').then(rows => {
let i = 0;
for (let row of rows) {
const cells = row.cells;
const requestDateStr = new Date(this.originSaveJSON.data[i].save_request_date).toLocaleString();
const saveStatus = this.originSaveJSON.data[i].save_task_status;
assert.equal($(cells[0]).text(), requestDateStr);
assert.equal($(cells[1]).text(), this.originSaveJSON.data[i].visit_type);
let html = '';
if (saveStatus === 'succeed') {
let browseOriginUrl = `${this.Urls.browse_origin()}?origin_url=${this.originSaveJSON.data[i].origin_url}`;
browseOriginUrl += `×tamp=${this.originSaveJSON.data[i].visit_date}`;
html += `${this.originSaveJSON.data[i].origin_url}`;
} else {
html += this.originSaveJSON.data[i].origin_url;
}
html += ` `;
html += '';
assert.equal($(cells[2]).html(), html);
assert.equal($(cells[3]).text(), this.originSaveJSON.data[i].save_request_status);
assert.equal($(cells[4]).text(), saveStatus);
++i;
}
});
});
it('should display/close task info popover when clicking on the info button', function() {
cy.fixture('origin-save').as('originSaveJSON');
cy.fixture('save-task-info').as('saveTaskInfoJSON');
cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON');
cy.get('#swh-origin-save-requests-list-tab').click();
cy.get('.swh-save-request-info')
.eq(0)
.click();
cy.get('.swh-save-request-info-popover')
.should('be.visible');
cy.get('.swh-save-request-info')
.eq(0)
.click();
cy.get('.swh-save-request-info-popover')
.should('not.be.visible');
});
it('should hide task info popover when clicking on the close button', function() {
cy.fixture('origin-save').as('originSaveJSON');
cy.fixture('save-task-info').as('saveTaskInfoJSON');
cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
cy.route('GET', '/save/task/info/**', '@saveTaskInfoJSON');
cy.get('#swh-origin-save-requests-list-tab').click();
cy.get('.swh-save-request-info')
.eq(0)
.click();
cy.get('.swh-save-request-info-popover')
.should('be.visible');
cy.get('.swh-save-request-info-close')
.click();
cy.get('.swh-save-request-info-popover')
.should('not.be.visible');
});
+ it('should fill save request form when clicking on "Save again" button', function() {
+ cy.fixture('origin-save').as('originSaveJSON');
+ cy.route('GET', '/save/requests/list/**', '@originSaveJSON');
+
+ cy.get('#swh-origin-save-requests-list-tab').click();
+ cy.get('.swh-save-origin-again')
+ .eq(0)
+ .click();
+
+ cy.get('tbody tr').eq(0).then(row => {
+ const cells = row[0].cells;
+ cy.get('#swh-input-visit-type')
+ .should('have.value', $(cells[1]).text());
+ cy.get('#swh-input-origin-url')
+ .should('have.value', $(cells[2]).text().slice(0, -1));
+ });
+ });
});
diff --git a/swh/web/assets/src/bundles/save/index.js b/swh/web/assets/src/bundles/save/index.js
index bb739c07..c298f63a 100644
--- a/swh/web/assets/src/bundles/save/index.js
+++ b/swh/web/assets/src/bundles/save/index.js
@@ -1,433 +1,457 @@
/**
* Copyright (C) 2018-2020 The Software Heritage developers
* See the AUTHORS file at the top-level directory of this distribution
* License: GNU Affero General Public License version 3, or any later version
* See top-level LICENSE file for more information
*/
import {handleFetchError, csrfPost, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions';
import {swhSpinnerSrc} from 'utils/constants';
import {validate} from 'validate.js';
let saveRequestsTable;
function originSaveRequest(originType, originUrl,
acceptedCallback, pendingCallback, errorCallback) {
let addSaveOriginRequestUrl = Urls.origin_save_request(originType, originUrl);
let headers = {
'Accept': 'application/json',
'Content-Type': 'application/json'
};
$('.swh-processing-save-request').css('display', 'block');
csrfPost(addSaveOriginRequestUrl, headers)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
$('.swh-processing-save-request').css('display', 'none');
if (data.save_request_status === 'accepted') {
acceptedCallback();
} else {
pendingCallback();
}
})
.catch(response => {
$('.swh-processing-save-request').css('display', 'none');
response.json().then(errorData => {
errorCallback(response.status, errorData);
});
});
}
export function initOriginSave() {
$(document).ready(() => {
$.fn.dataTable.ext.errMode = 'none';
fetch(Urls.origin_save_types_list())
.then(response => response.json())
.then(data => {
for (let originType of data) {
$('#swh-input-visit-type').append(``);
}
});
saveRequestsTable = $('#swh-origin-save-requests')
.on('error.dt', (e, settings, techNote, message) => {
$('#swh-origin-save-request-list-error').text('An error occurred while retrieving the save requests list');
console.log(message);
})
.DataTable({
serverSide: true,
processing: true,
language: {
processing: ``
},
ajax: Urls.origin_save_requests_list('all'),
searchDelay: 1000,
columns: [
{
data: 'save_request_date',
name: 'request_date',
render: (data, type, row) => {
if (type === 'display') {
let date = new Date(data);
return date.toLocaleString();
}
return data;
}
},
{
data: 'visit_type',
name: 'visit_type'
},
{
data: 'origin_url',
name: 'origin_url',
render: (data, type, row) => {
if (type === 'display') {
let html = '';
const sanitizedURL = $.fn.dataTable.render.text().display(data);
if (row.save_task_status === 'succeed') {
let browseOriginUrl = `${Urls.browse_origin()}?origin_url=${sanitizedURL}`;
browseOriginUrl += `×tamp=${row.visit_date}`;
html += `${sanitizedURL}`;
} else {
html += sanitizedURL;
}
html += ` `;
return html;
}
return data;
}
},
{
data: 'save_request_status',
name: 'status'
},
{
data: 'save_task_status',
name: 'loading_task_status'
},
{
name: 'info',
render: (data, type, row) => {
if (row.save_task_status === 'succeed' || row.save_task_status === 'failed') {
return ``;
} else {
return '';
}
}
+ },
+ {
+ render: (data, type, row) => {
+ if (row.save_request_status === 'accepted') {
+ const saveAgainButton =
+ '';
+ return saveAgainButton;
+ }
+ }
}
],
scrollY: '50vh',
scrollCollapse: true,
order: [[0, 'desc']],
responsive: {
details: {
type: 'none'
}
}
});
swh.webapp.addJumpToPagePopoverToDataTable(saveRequestsTable);
$('#swh-origin-save-requests-list-tab').on('shown.bs.tab', () => {
saveRequestsTable.draw();
window.location.hash = '#requests';
});
$('#swh-origin-save-request-help-tab').on('shown.bs.tab', () => {
removeUrlFragment();
$('.swh-save-request-info').popover('dispose');
});
let saveRequestAcceptedAlert = htmlAlert(
'success',
'The "save code now" request has been accepted and will be processed as soon as possible.',
true
);
let saveRequestPendingAlert = htmlAlert(
'warning',
'The "save code now" request has been put in pending state and may be accepted for processing after manual review.',
true
);
let saveRequestRateLimitedAlert = htmlAlert(
'danger',
'The rate limit for "save code now" requests has been reached. Please try again later.',
true
);
let saveRequestUnknownErrorAlert = htmlAlert(
'danger',
'An unexpected error happened when submitting the "save code now request".',
true
);
$('#swh-save-origin-form').submit(event => {
event.preventDefault();
event.stopPropagation();
$('.alert').alert('close');
if (event.target.checkValidity()) {
$(event.target).removeClass('was-validated');
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
originSaveRequest(originType, originUrl,
() => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert),
() => $('#swh-origin-save-request-status').html(saveRequestPendingAlert),
(statusCode, errorData) => {
$('#swh-origin-save-request-status').css('color', 'red');
if (statusCode === 403) {
const errorAlert = htmlAlert('danger', `Error: ${errorData['detail']}`);
$('#swh-origin-save-request-status').html(errorAlert);
} else if (statusCode === 429) {
$('#swh-origin-save-request-status').html(saveRequestRateLimitedAlert);
} else {
$('#swh-origin-save-request-status').html(saveRequestUnknownErrorAlert);
}
});
} else {
$(event.target).addClass('was-validated');
}
});
$('#swh-show-origin-save-requests-list').on('click', (event) => {
event.preventDefault();
$('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show');
});
$('#swh-input-origin-url').on('input', function(event) {
let originUrl = $(this).val().trim();
$(this).val(originUrl);
$('#swh-input-visit-type option').each(function() {
let val = $(this).val();
if (val && originUrl.includes(val)) {
$(this).prop('selected', true);
}
});
});
if (window.location.hash === '#requests') {
$('.nav-tabs a[href="#swh-origin-save-requests-list"]').tab('show');
}
});
}
export function validateSaveOriginUrl(input) {
let originUrl = input.value.trim();
let validUrl = validate({website: originUrl}, {
website: {
url: {
schemes: ['http', 'https', 'svn', 'git']
}
}
}) === undefined;
let originType = $('#swh-input-visit-type').val();
if (originType === 'git' && validUrl) {
// additional checks for well known code hosting providers
let githubIdx = originUrl.indexOf('://github.com');
let gitlabIdx = originUrl.indexOf('://gitlab.');
let gitSfIdx = originUrl.indexOf('://git.code.sf.net');
let bitbucketIdx = originUrl.indexOf('://bitbucket.org');
if (githubIdx !== -1 && githubIdx <= 5) {
validUrl = isGitRepoUrl(originUrl, 'github.com');
} else if (gitlabIdx !== -1 && gitlabIdx <= 5) {
let startIdx = gitlabIdx + 3;
let idx = originUrl.indexOf('/', startIdx);
if (idx !== -1) {
let gitlabDomain = originUrl.substr(startIdx, idx - startIdx);
validUrl = isGitRepoUrl(originUrl, gitlabDomain);
} else {
validUrl = false;
}
} else if (gitSfIdx !== -1 && gitSfIdx <= 5) {
validUrl = isGitRepoUrl(originUrl, 'git.code.sf.net/p');
} else if (bitbucketIdx !== -1 && bitbucketIdx <= 5) {
validUrl = isGitRepoUrl(originUrl, 'bitbucket.org');
}
}
if (validUrl) {
input.setCustomValidity('');
} else {
input.setCustomValidity('The origin url is not valid or does not reference a code repository');
}
}
export function initTakeNewSnapshot() {
let newSnapshotRequestAcceptedAlert = htmlAlert(
'success',
'The "take new snapshot" request has been accepted and will be processed as soon as possible.',
true
);
let newSnapshotRequestPendingAlert = htmlAlert(
'warning',
'The "take new snapshot" request has been put in pending state and may be accepted for processing after manual review.',
true
);
let newSnapshotRequestRateLimitAlert = htmlAlert(
'danger',
'The rate limit for "take new snapshot" requests has been reached. Please try again later.',
true
);
let newSnapshotRequestUnknownErrorAlert = htmlAlert(
'danger',
'An unexpected error happened when submitting the "save code now request".',
true
);
$(document).ready(() => {
$('#swh-take-new-snapshot-form').submit(event => {
event.preventDefault();
event.stopPropagation();
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
originSaveRequest(originType, originUrl,
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert),
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert),
(statusCode, errorData) => {
$('#swh-take-new-snapshot-request-status').css('color', 'red');
if (statusCode === 403) {
const errorAlert = htmlAlert('danger', `Error: ${errorData['detail']}`, true);
$('#swh-take-new-snapshot-request-status').html(errorAlert);
} else if (statusCode === 429) {
$('#swh-take-new-snapshot-request-status').html(newSnapshotRequestRateLimitAlert);
} else {
$('#swh-take-new-snapshot-request-status').html(newSnapshotRequestUnknownErrorAlert);
}
});
});
});
}
export function displaySaveRequestInfo(event, saveRequestId) {
event.stopPropagation();
const saveRequestTaskInfoUrl = Urls.origin_save_task_info(saveRequestId);
// close popover when clicking again on the info icon
if ($(event.target).data('bs.popover')) {
$(event.target).popover('dispose');
return;
}
$('.swh-save-request-info').popover('dispose');
$(event.target).popover({
animation: false,
boundary: 'viewport',
container: 'body',
title: 'Save request task information ' +
'`,
content: `
Fetching task information ...
${info.key} |
${info.value} |
---|
You can contribute to extend the content of the Software Heritage archive by submitting an origin save request. To do so, fill the required info in the form below:
A "Save code now" request takes the following parameters:
git
, for origins using Githg
, for origins using Mercurialsvn
, for origins using Subversiongit
origin into the archive, you should check that the command $ git clone <origin_url>
Once submitted, your save request can either be:
Once a save request has been accepted, you can follow its current status in the submitted save requests list.
Date | Type | Url | Request | Status | Info | +
---|