diff --git a/assets/src/bundles/vault/vault-create-tasks.js b/assets/src/bundles/vault/vault-create-tasks.js index 35f55961..d954eb35 100644 --- a/assets/src/bundles/vault/vault-create-tasks.js +++ b/assets/src/bundles/vault/vault-create-tasks.js @@ -1,171 +1,171 @@ /** * Copyright (C) 2018-2022 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ import * as EmailValidator from 'email-validator'; -import {handleFetchError, csrfPost, htmlAlert} from 'utils/functions'; +import {csrfPost, handleFetchError, htmlAlert} from 'utils/functions'; const alertStyle = { 'position': 'fixed', 'left': '1rem', 'bottom': '1rem', 'z-index': '100000' }; function vaultModalHandleEnterKey(event) { if (event.keyCode === 13) { event.preventDefault(); $('.modal.show').last().find('button:contains("Ok")').trigger('click'); } } export async function vaultRequest(objectType, swhid) { let vaultUrl; if (objectType === 'directory') { vaultUrl = Urls.api_1_vault_cook_flat(swhid); } else { vaultUrl = Urls.api_1_vault_cook_git_bare(swhid); } // check if object has already been cooked const response = await fetch(vaultUrl); const data = await response.json(); // object needs to be cooked if (data.exception === 'NotFoundExc' || data.status === 'failed') { // if last cooking has failed, remove previous task info from localStorage // in order to force the recooking of the object swh.vault.removeCookingTaskInfo([swhid]); const vaultModalId = `#vault-cook-${objectType}-modal`; $(vaultModalId).modal('show'); $('body').on('keyup', vaultModalId, vaultModalHandleEnterKey); // object has been cooked and should be in the vault cache, // it will be asked to cook it again if it is not } else if (data.status === 'done') { const vaultModalId = `#vault-fetch-${objectType}-modal`; $(vaultModalId).modal('show'); $('body').on('keyup', vaultModalId, vaultModalHandleEnterKey); } else { const cookingServiceDownAlert = $(htmlAlert('danger', 'Archive cooking service is currently experiencing issues.<br/>' + 'Please try again later.', true)); cookingServiceDownAlert.css(alertStyle); $('body').append(cookingServiceDownAlert); } } async function addVaultCookingTask(objectType, cookingTask) { const swhidsContext = swh.webapp.getSwhIdsContext(); cookingTask.origin = swhidsContext[objectType].context.origin; cookingTask.path = swhidsContext[objectType].context.path; cookingTask.browse_url = swhidsContext[objectType].swhid_with_context_url; if (!cookingTask.browse_url) { cookingTask.browse_url = swhidsContext[objectType].swhid_url; } let vaultCookingTasks = JSON.parse(localStorage.getItem('swh-vault-cooking-tasks')); if (!vaultCookingTasks) { vaultCookingTasks = []; } if (vaultCookingTasks.find(val => { return val.bundle_type === cookingTask.bundle_type && val.swhid === cookingTask.swhid; }) === undefined) { let cookingUrl; if (cookingTask.bundle_type === 'flat') { cookingUrl = Urls.api_1_vault_cook_flat(cookingTask.swhid); } else { cookingUrl = Urls.api_1_vault_cook_git_bare(cookingTask.swhid); } if (cookingTask.email) { cookingUrl += '?email=' + encodeURIComponent(cookingTask.email); } try { const response = await csrfPost(cookingUrl); handleFetchError(response); vaultCookingTasks.push(cookingTask); localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultCookingTasks)); $('#vault-cook-directory-modal').modal('hide'); $('body').off('keyup', '#vault-cook-directory-modal', vaultModalHandleEnterKey); $('#vault-cook-revision-modal').modal('hide'); $('body').off('keyup', '#vault-cook-revision-modal', vaultModalHandleEnterKey); const cookingTaskCreatedAlert = $(htmlAlert('success', 'Archive cooking request successfully submitted.<br/>' + - `Go to the <a href="${Urls.browse_vault()}">Downloads</a> page ` + + `Go to the <a href="${Urls.vault()}">Downloads</a> page ` + 'to get the download link once it is ready.', true)); cookingTaskCreatedAlert.css(alertStyle); $('body').append(cookingTaskCreatedAlert); } catch (_) { $('#vault-cook-directory-modal').modal('hide'); $('body').off('keyup', '#vault-cook-directory-modal', vaultModalHandleEnterKey); $('#vault-cook-revision-modal').modal('hide'); $('body').off('keyup', '#vault-cook-revision-modal', vaultModalHandleEnterKey); const cookingTaskFailedAlert = $(htmlAlert('danger', 'Archive cooking request submission failed.', true)); cookingTaskFailedAlert.css(alertStyle); $('body').append(cookingTaskFailedAlert); } } } export function cookDirectoryArchive(swhid) { const email = $('#swh-vault-directory-email').val().trim(); if (!email || EmailValidator.validate(email)) { const cookingTask = { 'bundle_type': 'flat', 'swhid': swhid, 'email': email, 'status': 'new' }; addVaultCookingTask('directory', cookingTask); } else { $('#invalid-email-modal').modal('show'); $('body').on('keyup', '#invalid-email-modal', vaultModalHandleEnterKey); } } export async function fetchDirectoryArchive(directorySwhid) { $('#vault-fetch-directory-modal').modal('hide'); $('body').off('keyup', '#vault-cook-revision-modal', vaultModalHandleEnterKey); const vaultUrl = Urls.api_1_vault_cook_flat(directorySwhid); const response = await fetch(vaultUrl); const data = await response.json(); swh.vault.fetchCookedObject(data.fetch_url); } export function cookRevisionArchive(revisionId) { const email = $('#swh-vault-revision-email').val().trim(); if (!email || EmailValidator.validate(email)) { const cookingTask = { 'bundle_type': 'git_bare', 'swhid': revisionId, 'email': email, 'status': 'new' }; addVaultCookingTask('revision', cookingTask); } else { $('#invalid-email-modal').modal('show'); $('body').on('keyup', '#invalid-email-modal', vaultModalHandleEnterKey); } } export async function fetchRevisionArchive(revisionSwhid) { $('#vault-fetch-revision-modal').modal('hide'); $('body').off('keyup', '#vault-fetch-revision-modal', vaultModalHandleEnterKey); const vaultUrl = Urls.api_1_vault_cook_git_bare(revisionSwhid); const response = await fetch(vaultUrl); const data = await response.json(); swh.vault.fetchCookedObject(data.fetch_url); } diff --git a/cypress/e2e/vault.cy.js b/cypress/e2e/vault.cy.js index 9c98efff..1cd13231 100644 --- a/cypress/e2e/vault.cy.js +++ b/cypress/e2e/vault.cy.js @@ -1,580 +1,580 @@ /** * Copyright (C) 2019-2022 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ const progressbarColors = { 'new': 'rgba(128, 128, 128, 0.5)', 'pending': 'rgba(0, 0, 255, 0.5)', 'done': 'rgb(92, 184, 92)' }; function checkVaultCookingTask(objectType) { cy.contains('button', 'Download') .click(); cy.contains('.dropdown-item', objectType) .click(); cy.wait('@checkVaultCookingTask'); } function getVaultItemList() { return JSON.parse(window.localStorage.getItem('swh-vault-cooking-tasks')); } function updateVaultItemList(vaultItems) { window.localStorage.setItem('swh-vault-cooking-tasks', JSON.stringify(vaultItems)); } // Mocks API response : /api/1/vault/(:bundleType)/(:swhid) // bundleType : {'flat', 'git_bare'} function genVaultCookingResponse(bundleType, swhid, status, message, fetchUrl) { return { 'bundle_type': bundleType, 'id': 1, 'progress_message': message, 'status': status, 'swhid': swhid, 'fetch_url': fetchUrl }; }; // Tests progressbar color, status // And status in localStorage function testStatus(taskId, color, statusMsg, status) { cy.get(`.swh-vault-table #vault-task-${CSS.escape(taskId)}`) .should('be.visible') .find('.progress-bar') .should('be.visible') .and('have.css', 'background-color', color) .and('contain', statusMsg) .then(() => { // Vault item with object_id as taskId should exist in localStorage const currentVaultItems = getVaultItemList(); const vaultItem = currentVaultItems.find(obj => obj.swhid === taskId); assert.isNotNull(vaultItem); assert.strictEqual(vaultItem.status, status); }); } describe('Vault Cooking User Interface Tests', function() { before(function() { const dirInfo = this.origin[0].directory[0]; this.directory = `swh:1:dir:${dirInfo.id}`; this.directoryUrl = this.Urls.browse_origin_directory() + `?origin_url=${this.origin[0].url}&path=${dirInfo.path}`; this.vaultDirectoryUrl = this.Urls.api_1_vault_cook_flat(this.directory); this.vaultFetchDirectoryUrl = this.Urls.api_1_vault_fetch_flat(this.directory); this.revisionId = this.origin[1].revisions[0]; this.revision = `swh:1:rev:${this.revisionId}`; this.revisionUrl = this.Urls.browse_revision(this.revisionId); this.vaultRevisionUrl = this.Urls.api_1_vault_cook_git_bare(this.revision); this.vaultFetchRevisionUrl = this.Urls.api_1_vault_fetch_git_bare(this.revision); const release = this.origin[1].release; this.releaseUrl = this.Urls.browse_release(release.id) + `?origin_url=${this.origin[1].url}`; this.vaultReleaseDirectoryUrl = this.Urls.api_1_vault_cook_flat(`swh:1:dir:${release.directory}`); }); beforeEach(function() { // For some reason, this gets reset if we define it in the before() hook, // so we need to define it here this.vaultItems = [ { 'bundle_type': 'git_bare', 'swhid': this.revision, 'email': '', 'status': 'done', 'fetch_url': `/api/1/vault/git-bare/${this.revision}/raw/`, 'progress_message': null } ]; this.legacyVaultItems = [ { 'object_type': 'revision', 'object_id': this.revisionId, 'email': '', 'status': 'done', 'fetch_url': `/api/1/vault/revision/${this.revisionId}/gitfast/raw/`, 'progress_message': null } ]; this.genVaultDirCookingResponse = (status, message = null) => { return genVaultCookingResponse('flat', this.directory, status, message, this.vaultFetchDirectoryUrl); }; this.genVaultRevCookingResponse = (status, message = null) => { return genVaultCookingResponse('git_bare', this.revision, status, message, this.vaultFetchRevisionUrl); }; }); it('should report an error when vault service is experiencing issues', function() { // Browse a directory cy.visit(this.directoryUrl); // Stub responses when requesting the vault API to simulate // an internal server error cy.intercept(this.vaultDirectoryUrl, { body: {'exception': 'APIError'}, statusCode: 500 }).as('checkVaultCookingTask'); cy.contains('button', 'Download') .click(); // Check error alert is displayed cy.get('.alert-danger') .should('be.visible') .should('contain', 'Archive cooking service is currently experiencing issues.'); }); it('should report an error when a cooking task creation failed', function() { // Browse a directory cy.visit(this.directoryUrl); // Stub responses when requesting the vault API to simulate // a task can not be created cy.intercept('GET', this.vaultDirectoryUrl, { body: {'exception': 'NotFoundExc'} }).as('checkVaultCookingTask'); cy.intercept('POST', this.vaultDirectoryUrl, { body: {'exception': 'ValueError'}, statusCode: 500 }).as('createVaultCookingTask'); cy.contains('button', 'Download') .click(); // Create a vault cooking task through the GUI cy.get('.modal-dialog') .contains('button:visible', 'Ok') .click(); cy.wait('@createVaultCookingTask'); // Check error alert is displayed cy.get('.alert-danger') .should('be.visible') .should('contain', 'Archive cooking request submission failed.'); }); it('should display previous cooking tasks', function() { updateVaultItemList(this.vaultItems); - cy.visit(this.Urls.browse_vault()); + cy.visit(this.Urls.vault()); cy.contains(`#vault-task-${CSS.escape(this.revision)} button`, 'Download') .click(); }); it('should display and upgrade previous cooking tasks from the legacy format', function() { updateVaultItemList(this.legacyVaultItems); - cy.visit(this.Urls.browse_vault()); + cy.visit(this.Urls.vault()); // Check it is displayed cy.contains(`#vault-task-${CSS.escape(this.revision)} button`, 'Download') .then(() => { // Check the LocalStorage was upgraded expect(getVaultItemList()).to.deep.equal(this.vaultItems); }); }); it('should create a directory cooking task and report the success', function() { // Browse a directory cy.visit(this.directoryUrl); // Stub response to the vault API to simulate archive download cy.intercept('GET', this.vaultFetchDirectoryUrl, { fixture: `${this.directory.replace(/:/g, '_')}.tar.gz`, headers: { 'Content-disposition': `attachment; filename=${this.directory.replace(/:/g, '_')}.tar.gz`, 'Content-Type': 'application/gzip' } }).as('fetchCookedArchive'); // Stub responses when checking vault task status const checkVaulResponses = [ {'exception': 'NotFoundExc'}, this.genVaultDirCookingResponse('new'), this.genVaultDirCookingResponse('pending', 'Processing...'), this.genVaultDirCookingResponse('done') ]; // trick to override the response of an intercepted request // https://github.com/cypress-io/cypress/issues/9302 cy.intercept('GET', this.vaultDirectoryUrl, req => req.reply(checkVaulResponses.shift())) .as('checkVaultCookingTask'); // Stub responses when requesting the vault API to simulate // a task has been created cy.intercept('POST', this.vaultDirectoryUrl, { body: this.genVaultDirCookingResponse('new') }).as('createVaultCookingTask'); cy.contains('button', 'Download') .click(); cy.window().then(win => { const swhIdsContext = win.swh.webapp.getSwhIdsContext(); const browseDirectoryUrl = swhIdsContext.directory.swhid_with_context_url; // Create a vault cooking task through the GUI cy.get('.modal-dialog') .contains('button:visible', 'Ok') .click(); cy.wait('@createVaultCookingTask'); // Check success alert is displayed cy.get('.alert-success') .should('be.visible') .should('contain', 'Archive cooking request successfully submitted.'); // Go to Downloads page - cy.visit(this.Urls.browse_vault()); + cy.visit(this.Urls.vault()); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.directory, progressbarColors['new'], 'new', 'new'); }); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.directory, progressbarColors['pending'], 'Processing...', 'pending'); }); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.directory, progressbarColors['done'], 'done', 'done'); }); cy.get(`#vault-task-${CSS.escape(this.directory)} .vault-origin a`) .should('contain', this.origin[0].url) .should('have.attr', 'href', `${this.Urls.browse_origin()}?origin_url=${this.origin[0].url}`); cy.get(`#vault-task-${CSS.escape(this.directory)} .vault-object-info a`) .should('have.text', this.directory) .should('have.attr', 'href', browseDirectoryUrl); cy.get(`#vault-task-${CSS.escape(this.directory)}`) .invoke('attr', 'title') .should('contain', 'the directory can be extracted'); cy.get(`#vault-task-${CSS.escape(this.directory)} .vault-dl-link button`) .click(); cy.wait('@fetchCookedArchive').then((xhr) => { assert.isNotNull(xhr.response.body); }); }); }); it('should create a revision cooking task and report its status', function() { cy.adminLogin(); // Browse a revision cy.visit(this.revisionUrl); // Stub response to the vault API indicating to simulate archive download cy.intercept({url: this.vaultFetchRevisionUrl}, { fixture: `${this.revision.replace(/:/g, '_')}.git.tar`, headers: { 'Content-disposition': `attachment; filename=${this.revision.replace(/:/g, '_')}.git.tar`, 'Content-Type': 'application/x-tar' } }).as('fetchCookedArchive'); // Stub responses when checking vault task status const checkVaultResponses = [ {'exception': 'NotFoundExc'}, this.genVaultRevCookingResponse('new'), this.genVaultRevCookingResponse('pending', 'Processing...'), this.genVaultRevCookingResponse('done') ]; // trick to override the response of an intercepted request // https://github.com/cypress-io/cypress/issues/9302 cy.intercept('GET', this.vaultRevisionUrl, req => req.reply(checkVaultResponses.shift())) .as('checkVaultCookingTask'); // Stub responses when requesting the vault API to simulate // a task has been created cy.intercept('POST', this.vaultRevisionUrl, { body: this.genVaultRevCookingResponse('new') }).as('createVaultCookingTask'); // Create a vault cooking task through the GUI checkVaultCookingTask('as git'); cy.window().then(win => { const swhIdsContext = win.swh.webapp.getSwhIdsContext(); const browseRevisionUrl = swhIdsContext.revision.swhid_url; // Create a vault cooking task through the GUI cy.get('.modal.show') .type('{enter}'); cy.wait('@createVaultCookingTask'); // Check success alert is displayed cy.get('.alert-success') .should('be.visible') .should('contain', 'Archive cooking request successfully submitted.'); // Go to Downloads page - cy.visit(this.Urls.browse_vault()); + cy.visit(this.Urls.vault()); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.revision, progressbarColors['new'], 'new', 'new'); }); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.revision, progressbarColors['pending'], 'Processing...', 'pending'); }); cy.wait('@checkVaultCookingTask').then(() => { testStatus(this.revision, progressbarColors['done'], 'done', 'done'); }); cy.get(`#vault-task-${CSS.escape(this.revision)} .vault-origin`) .should('have.text', 'unknown'); cy.get(`#vault-task-${CSS.escape(this.revision)} .vault-object-info a`) .should('have.text', this.revision) .should('have.attr', 'href', browseRevisionUrl); cy.get(`#vault-task-${CSS.escape(this.revision)}`) .invoke('attr', 'title') .should('contain', 'the git repository can be imported'); cy.get(`#vault-task-${CSS.escape(this.revision)} .vault-dl-link button`) .click(); cy.wait('@fetchCookedArchive').then((xhr) => { assert.isNotNull(xhr.response.body); }); }); }); it('should create a directory cooking task from the release view', function() { // Browse a directory cy.visit(this.releaseUrl); // Stub responses when checking vault task status const checkVaultResponses = [ {'exception': 'NotFoundExc'}, this.genVaultDirCookingResponse('new') ]; // trick to override the response of an intercepted request // https://github.com/cypress-io/cypress/issues/9302 cy.intercept('GET', this.vaultReleaseDirectoryUrl, req => req.reply(checkVaultResponses.shift())) .as('checkVaultCookingTask'); // Stub responses when requesting the vault API to simulate // a task has been created cy.intercept('POST', this.vaultReleaseDirectoryUrl, { body: this.genVaultDirCookingResponse('new') }).as('createVaultCookingTask'); cy.contains('button', 'Download') .click(); // Create a vault cooking task through the GUI cy.get('.modal-dialog') .contains('button:visible', 'Ok') .click(); cy.wait('@createVaultCookingTask'); // Check success alert is displayed cy.get('.alert-success') .should('be.visible') .should('contain', 'Archive cooking request successfully submitted.'); }); it('should create a directory cooking task with an email address', function() { // Browse a directory cy.visit(this.directoryUrl); // Stub responses when checking vault task status cy.intercept('GET', this.vaultDirectoryUrl, {body: {'exception': 'NotFoundExc'}}) .as('checkVaultCookingTask'); // Stub responses when requesting the vault API to simulate // a task has been created cy.intercept('POST', this.vaultDirectoryUrl + '?email=foo%2Bbar%40example.org', { body: this.genVaultDirCookingResponse('new') }).as('createVaultCookingTask'); // Open vault cook directory modal cy.contains('button', 'Download') .click(); cy.wait('@checkVaultCookingTask'); // Create a vault cooking task through the GUI and fill email input cy.get('#vault-cook-directory-modal input[type="email"]') .type('foo+bar@example.org', {force: true}); cy.get('#vault-cook-directory-modal') .contains('button:visible', 'Ok') .click(); cy.wait('@createVaultCookingTask'); }); it('should offer to recook an archive if no longer available for download', function() { updateVaultItemList(this.vaultItems); // Send 404 when fetching vault item cy.intercept({url: this.vaultFetchRevisionUrl}, { statusCode: 404, body: { 'exception': 'NotFoundExc', 'reason': `Revision with ID '${this.revision}' not found.` }, headers: { 'Content-Type': 'json' } }).as('fetchCookedArchive'); - cy.visit(this.Urls.browse_vault()) + cy.visit(this.Urls.vault()) .get(`#vault-task-${CSS.escape(this.revision)} .vault-dl-link button`) .click(); cy.wait('@fetchCookedArchive').then(() => { cy.intercept('POST', this.vaultRevisionUrl, { body: this.genVaultRevCookingResponse('new') }).as('createVaultCookingTask'); cy.intercept(this.vaultRevisionUrl, { body: this.genVaultRevCookingResponse('new') }).as('checkVaultCookingTask'); cy.get('#vault-recook-object-modal > .modal-dialog') .should('be.visible') .contains('button:visible', 'Ok') .click(); cy.wait('@checkVaultCookingTask') .then(() => { testStatus(this.revision, progressbarColors['new'], 'new', 'new'); }); }); }); it('should remove selected vault items', function() { updateVaultItemList(this.vaultItems); - cy.visit(this.Urls.browse_vault()) + cy.visit(this.Urls.vault()) .get(`#vault-task-${CSS.escape(this.revision)}`) .find('input[type="checkbox"]') .click({force: true}); cy.contains('button', 'Remove selected tasks') .click(); cy.get(`#vault-task-${CSS.escape(this.revision)}`) .should('not.exist'); }); it('should offer to immediately download a directory tarball if already cooked', function() { // Browse a directory cy.visit(this.directoryUrl); // Stub response to the vault API to simulate archive download cy.intercept({url: this.vaultFetchDirectoryUrl}, { fixture: `${this.directory.replace(/:/g, '_')}.tar.gz`, headers: { 'Content-disposition': `attachment; filename=${this.directory.replace(/:/g, '_')}.tar.gz`, 'Content-Type': 'application/gzip' } }).as('fetchCookedArchive'); // Stub responses when requesting the vault API to simulate // the directory tarball has already been cooked cy.intercept(this.vaultDirectoryUrl, { body: this.genVaultDirCookingResponse('done') }).as('checkVaultCookingTask'); // Create a vault cooking task through the GUI cy.contains('button', 'Download') .click(); // Start archive download through the GUI cy.get('.modal-dialog') .contains('button:visible', 'Ok') .click(); cy.wait('@fetchCookedArchive'); }); it('should offer to immediately download a bare revision git archive if already cooked', function() { cy.adminLogin(); // Browse a directory cy.visit(this.revisionUrl); // Stub response to the vault API to simulate archive download cy.intercept({url: this.vaultFetchRevisionUrl}, { fixture: `${this.revision.replace(/:/g, '_')}.git.tar`, headers: { 'Content-disposition': `attachment; filename=${this.revision.replace(/:/g, '_')}.git.tar`, 'Content-Type': 'application/x-tar' } }).as('fetchCookedArchive'); // Stub responses when requesting the vault API to simulate // the directory tarball has already been cooked cy.intercept(this.vaultRevisionUrl, { body: this.genVaultRevCookingResponse('done') }).as('checkVaultCookingTask'); checkVaultCookingTask('as git'); // Start archive download through the GUI cy.get('.modal-dialog') .contains('button:visible', 'Ok') .click(); cy.wait('@fetchCookedArchive'); }); it('should offer to recook an object if previous vault task failed', function() { cy.visit(this.directoryUrl); // Stub responses when requesting the vault API to simulate // the last cooking of the directory tarball has failed cy.intercept(this.vaultDirectoryUrl, { body: this.genVaultDirCookingResponse('failed') }).as('checkVaultCookingTask'); cy.contains('button', 'Download') .click(); // Check that recooking the directory is offered to user cy.get('.modal-dialog') .contains('button:visible', 'Ok') .should('be.visible'); }); }); diff --git a/docs/uri-scheme-api-vault.rst b/docs/uri-scheme-api-vault.rst index 765e7904..1f191520 100644 --- a/docs/uri-scheme-api-vault.rst +++ b/docs/uri-scheme-api-vault.rst @@ -1,10 +1,10 @@ Vault ----- -.. autosimple:: swh.web.api.views.vault.api_vault_cook_directory +.. autosimple:: swh.web.vault.api_views.api_vault_cook_directory -.. autosimple:: swh.web.api.views.vault.api_vault_fetch_directory +.. autosimple:: swh.web.vault.api_views.api_vault_fetch_directory -.. autosimple:: swh.web.api.views.vault.api_vault_cook_revision_gitfast +.. autosimple:: swh.web.vault.api_views.api_vault_cook_revision_gitfast -.. autosimple:: swh.web.api.views.vault.api_vault_fetch_revision_gitfast +.. autosimple:: swh.web.vault.api_views.api_vault_fetch_revision_gitfast diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py index 9fe9f56f..038e7e1c 100644 --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -1,22 +1,21 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.api.apiurls import APIUrls import swh.web.api.views.content # noqa import swh.web.api.views.directory # noqa import swh.web.api.views.graph # noqa import swh.web.api.views.identifiers # noqa import swh.web.api.views.metadata # noqa import swh.web.api.views.origin # noqa import swh.web.api.views.ping # noqa import swh.web.api.views.raw # noqa import swh.web.api.views.release # noqa import swh.web.api.views.revision # noqa import swh.web.api.views.snapshot # noqa import swh.web.api.views.stat # noqa -import swh.web.api.views.vault # noqa urlpatterns = APIUrls.get_url_patterns() diff --git a/swh/web/browse/templates/includes/top-navigation.html b/swh/web/browse/templates/includes/top-navigation.html index b28fd4ee..dabf76eb 100644 --- a/swh/web/browse/templates/includes/top-navigation.html +++ b/swh/web/browse/templates/includes/top-navigation.html @@ -1,155 +1,157 @@ {% comment %} Copyright (C) 2017-2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} <div class="swh-browse-top-navigation d-flex align-items-start justify-content-between flex-wrap mt-1"> {% if snapshot_context %} {% if snapshot_context.branch or snapshot_context.release or snapshot_context.revision_id %} <div class="dropdown" id="swh-branches-releases-dd"> <button class="btn btn-block btn-default btn-sm dropdown-toggle" type="button" data-toggle="dropdown"> {% if snapshot_context.branch %} {% if snapshot_context.branch_alias %} <i class="{{ swh_object_icons.alias }} mdi-fw" aria-hidden="true"></i> {% else %} <i class="{{ swh_object_icons.branch }} mdi-fw" aria-hidden="true"></i> {% endif %} Branch: <strong>{{ snapshot_context.branch }}</strong> {% elif snapshot_context.release %} {% if snapshot_context.release_alias %} <i class="{{ swh_object_icons.alias }} mdi-fw" aria-hidden="true"></i> {% else %} <i class="{{ swh_object_icons.release }} mdi-fw" aria-hidden="true"></i> {% endif %} Release: <strong>{{ snapshot_context.release }}</strong> {% elif snapshot_context.revision_id %} Revision: <strong>{{ snapshot_context.revision_id }}</strong> {% endif %} <span class="caret"></span> </button> <ul class="scrollable-menu dropdown-menu swh-branches-releases"> <ul class="nav nav-tabs"> <li class="nav-item"><a class="nav-link active swh-branches-switch" data-toggle="tab">Branches</a></li> <li class="nav-item"><a class="nav-link swh-releases-switch" data-toggle="tab">Releases</a></li> </ul> <div class="tab-content"> <div class="tab-pane active" id="swh-tab-branches"> {% for b in snapshot_context.branches %} <li class="swh-branch"> <a href="{{ b.url | safe }}"> {% if b.alias %} <i class="{{ swh_object_icons.alias }} mdi-fw" aria-hidden="true"></i> {% else %} <i class="{{ swh_object_icons.branch }} mdi-fw" aria-hidden="true"></i> {% endif %} {% if b.name == snapshot_context.branch %} <i class="mdi mdi-check-bold mdi-fw" aria-hidden="true"></i> {% else %} <i class="mdi mdi-fw" aria-hidden="true"></i> {% endif %} {{ b.name }} </a> </li> {% endfor %} {% if snapshot_context.branches|length < snapshot_context.snapshot_sizes.revision %} <li> <i class="mdi mdi-alert mdi-fw" aria-hidden="true"></i> Branches list truncated to {{ snapshot_context.branches|length }} entries, {{ snapshot_context.branches|length|mul:-1|add:snapshot_context.snapshot_sizes.revision }} were omitted. </li> {% endif %} </div> <div class="tab-pane" id="swh-tab-releases"> {% if snapshot_context.releases %} {% for r in snapshot_context.releases %} {% if r.target_type == 'revision' or r.target_type == 'directory' %} <li class="swh-release"> <a href="{{ r.url | safe }}"> {% if r.alias %} <i class="{{ swh_object_icons.alias }} mdi-fw" aria-hidden="true"></i> {% else %} <i class="{{ swh_object_icons.release }} mdi-fw" aria-hidden="true"></i> {% endif %} {% if r.name == snapshot_context.release %} <i class="mdi mdi-check-bold mdi-fw" aria-hidden="true"></i> {% else %} <i class="mdi mdi-fw" aria-hidden="true"></i> {% endif %} {{ r.name }} </a> </li> {% endif %} {% endfor %} {% if snapshot_context.releases|length < snapshot_context.snapshot_sizes.release %} <li> <i class="mdi mdi-alert mdi-fw" aria-hidden="true"></i> Releases list truncated to {{ snapshot_context.releases|length }} entries, {{ snapshot_context.releases|length|mul:-1|add:snapshot_context.snapshot_sizes.release }} were omitted. </li> {% endif %} {% else %} <span>No releases to show</span> {% endif %} </div> </div> </ul> </div> {% endif %} {% endif %} <div id="swh-breadcrumbs-container" class="flex-grow-1"> {% include "./breadcrumbs.html" %} </div> <div class="btn-group swh-actions-dropdown ml-auto"> {% if top_right_link %} <a href="{{ top_right_link.url | safe }}" class="btn btn-default btn-sm swh-tr-link" role="button"> {% if top_right_link.icon %} <i class="{{ top_right_link.icon }} mdi-fw" aria-hidden="true"></i> {% endif %} {{ top_right_link.text }} </a> {% endif %} {% if available_languages %} <select data-placeholder="Select Language" class="language-select chosen-select"> <option value=""></option> {% for lang in available_languages %} <option value="{{ lang }}">{{ lang }}</option> {% endfor %} </select> {% endif %} {% if show_actions %} - {% if not snapshot_context or not snapshot_context.is_empty %} - {% include "./vault-create-tasks.html" %} + {% if "swh.web.vault" in SWH_DJANGO_APPS %} + {% if not snapshot_context or not snapshot_context.is_empty %} + {% include "includes/vault-create-tasks.html" %} + {% endif %} {% endif %} {% if "swh.web.save_code_now" in SWH_DJANGO_APPS %} {% include "./take-new-snapshot.html" %} {% endif %} {% include "./show-metadata.html" %} {% endif %} </div> </div> {% include "./show-swhids.html" %} <script> var snapshotContext = false; var branch = false; {% if snapshot_context %} snapshotContext = true; branch = "{{ snapshot_context.branch|escape }}"; {% endif %} {% if available_languages %} $(".chosen-select").val("{{ language }}"); $(".chosen-select").chosen().change(function(event, params) { updateLanguage(params.selected); }); {% endif %} swh.browse.initSnapshotNavigation(snapshotContext, branch !== "None"); </script> diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py index 38b6b93b..7665ab8b 100644 --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -1,74 +1,65 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect, render from django.urls import re_path as url from swh.web.browse.browseurls import BrowseUrls from swh.web.browse.identifiers import swhid_browse import swh.web.browse.views.content # noqa import swh.web.browse.views.directory # noqa import swh.web.browse.views.iframe # noqa import swh.web.browse.views.origin # noqa import swh.web.browse.views.release # noqa import swh.web.browse.views.revision # noqa import swh.web.browse.views.snapshot # noqa from swh.web.utils import origin_visit_types, reverse def _browse_help_view(request: HttpRequest) -> HttpResponse: return render( request, "browse-help.html", {"heading": "How to browse the archive ?"} ) def _browse_search_view(request: HttpRequest) -> HttpResponse: return render( request, "browse-search.html", { "heading": "Search software origins to browse", "visit_types": origin_visit_types(), }, ) -def _browse_vault_view(request: HttpRequest) -> HttpResponse: - return render( - request, - "browse-vault-ui.html", - {"heading": "Download archive content from the Vault"}, - ) - - def _browse_origin_save_view(request: HttpRequest) -> HttpResponse: return redirect(reverse("origin-save")) def _browse_swhid_iframe_legacy(request: HttpRequest, swhid: str) -> HttpResponse: return redirect(reverse("browse-swhid-iframe", url_args={"swhid": swhid})) urlpatterns = [ url(r"^browse/$", _browse_search_view), url(r"^browse/help/$", _browse_help_view, name="browse-help"), url(r"^browse/search/$", _browse_search_view, name="browse-search"), - url(r"^browse/vault/$", _browse_vault_view, name="browse-vault"), # for backward compatibility url(r"^browse/origin/save/$", _browse_origin_save_view, name="browse-origin-save"), url( r"^browse/(?P<swhid>swh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$", swhid_browse, name="browse-swhid-legacy", ), url( r"^embed/(?P<swhid>swh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$", _browse_swhid_iframe_legacy, name="browse-swhid-iframe-legacy", ), ] urlpatterns += BrowseUrls.get_url_patterns() diff --git a/swh/web/config.py b/swh/web/config.py index 1f4377d9..d1703967 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,240 +1,241 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict from swh.core import config from swh.counters import get_counters from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SWH_WEB_SERVER_NAME = "archive.softwareheritage.org" SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org" SWH_WEB_STAGING_SERVER_NAMES = [ "webapp.staging.swh.network", "webapp.internal.staging.swh.network", ] SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { "allowed_hosts": ("list", []), "storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10, }, ), "indexer_storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1, }, ), "counters": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5011/", "timeout": 1, }, ), "search": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5010/", "timeout": 10, }, ), "search_config": ( "dict", { "metadata_backend": "swh-indexer-storage", }, # or "swh-search" ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), "serve_assets": ("bool", False), "host": ("string", "127.0.0.1"), "port": ("int", 5004), "secret_key": ("string", "development key"), # do not display code highlighting for content > 1MB "content_display_max_size": ("int", 5 * 1024 * 1024), "snapshot_content_max_size": ("int", 1000), "throttling": ( "dict", { "cache_uri": None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None "scopes": { "swh_api": { "limiter_rate": {"default": "120/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_search": { "limiter_rate": {"default": "10/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_vault_cooking": { "limiter_rate": {"default": "120/h", "GET": "60/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_save_origin": { "limiter_rate": {"default": "120/h", "POST": "10/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_visit_latest": { "limiter_rate": {"default": "700/m"}, "exempted_networks": ["127.0.0.0/8"], }, }, }, ), "vault": ( "dict", { "cls": "remote", "args": { "url": "http://127.0.0.1:5005/", }, }, ), "scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}), "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")), "test_db": ("dict", {"name": "swh-web-test"}), "production_db": ("dict", {"name": "swh-web"}), "deposit": ( "dict", { "private_api_url": "https://deposit.softwareheritage.org/1/private/", "private_api_user": "swhworker", "private_api_password": "some-password", }, ), "e2e_tests_mode": ("bool", False), "es_workers_index_url": ("string", ""), "history_counters_url": ( "string", ( "http://counters1.internal.softwareheritage.org:5011" "/counters_history/history.json" ), ), "client_config": ("dict", {}), "keycloak": ("dict", {"server_url": "", "realm_name": ""}), "graph": ( "dict", { "server_url": "http://graph.internal.softwareheritage.org:5009/graph/", "max_edges": {"staff": 0, "user": 100000, "anonymous": 1000}, }, ), "status": ( "dict", { "server_url": "https://status.softwareheritage.org/", "json_path": "1.0/status/578e5eddcdc0cc7951000520", }, ), "counters_backend": ("string", "swh-storage"), # or "swh-counters" "staging_server_names": ("list", SWH_WEB_STAGING_SERVER_NAMES), "instance_name": ("str", "archive-test.softwareheritage.org"), "give": ("dict", {"public_key": "", "token": ""}), "features": ("dict", {"add_forge_now": True}), "add_forge_now": ("dict", {"email_address": "add-forge-now@example.com"}), "swh_extra_django_apps": ( "list", [ "swh.web.inbound_email", "swh.web.add_forge_now", "swh.web.mailmap", "swh.web.save_code_now", "swh.web.deposit", "swh.web.badges", "swh.web.archive_coverage", "swh.web.metrics", "swh.web.banners", "swh.web.jslicenses", + "swh.web.vault", ], ), } swhweb_config: Dict[str, Any] = {} def get_config(config_file="web/web"): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get("SWH_CONFIG_FILENAME") if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, "log_dir") if swhweb_config.get("search"): swhweb_config["search"] = get_search(**swhweb_config["search"]) else: swhweb_config["search"] = None swhweb_config["storage"] = get_storage(**swhweb_config["storage"]) swhweb_config["vault"] = get_vault(**swhweb_config["vault"]) swhweb_config["indexer_storage"] = get_indexer_storage( **swhweb_config["indexer_storage"] ) swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) swhweb_config["counters"] = get_counters(**swhweb_config["counters"]) return swhweb_config def search(): """Return the current application's search.""" return get_config()["search"] def storage(): """Return the current application's storage.""" return get_config()["storage"] def vault(): """Return the current application's vault.""" return get_config()["vault"] def indexer_storage(): """Return the current application's indexer storage.""" return get_config()["indexer_storage"] def scheduler(): """Return the current application's scheduler.""" return get_config()["scheduler"] def counters(): """Return the current application's counters.""" return get_config()["counters"] diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index 479d3a79..5795a0b2 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,327 +1,329 @@ {% comment %} Copyright (C) 2015-2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} <!DOCTYPE html> {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} <html lang="en"> <head> <meta charset="utf-8"> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <title>{% block title %}{% endblock %}</title> {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% render_bundle 'guided_tour' %} <script> /* @licstart The following is the entire license notice for the JavaScript code in this page. Copyright (C) 2015-2022 The Software Heritage developers This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see <https://www.gnu.org/licenses/>. @licend The above is the entire license notice for the JavaScript code in this page. */ </script> <script> SWH_CONFIG = {{swh_client_config|jsonify}}; swh.webapp.sentryInit(SWH_CONFIG.sentry_dsn); </script> <script src="{% url 'js_reverse' %}" type="text/javascript"></script> <script> swh.webapp.setSwhObjectIcons({{ swh_object_icons|jsonify }}); </script> {{ request.user.is_authenticated|json_script:"swh_user_logged_in" }} {% include "includes/favicon.html" %} {% block header %}{% endblock %} {% if swh_web_prod %} <!-- Matomo --> <script type="text/javascript"> var _paq = window._paq = window._paq || []; _paq.push(['trackPageView']); (function() { var u="https://piwik.inria.fr/"; _paq.push(['setTrackerUrl', u+'matomo.php']); _paq.push(['setSiteId', '59']); var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0]; g.type='text/javascript'; g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s); })(); </script> <!-- End Matomo Code --> {% endif %} </head> <body class="hold-transition layout-fixed sidebar-mini {% if sidebar_state == 'collapsed' %} sidebar-collapse {% endif %}"> <a id="top"></a> <div class="wrapper"> <div class="swh-top-bar"> <ul> <li class="swh-position-left"> <div id="swh-full-width-switch-container" class="custom-control custom-switch d-none d-lg-block d-xl-block"> <input type="checkbox" class="custom-control-input" id="swh-full-width-switch" onclick="swh.webapp.fullWidthToggled(event)"> <label class="custom-control-label font-weight-normal" for="swh-full-width-switch">Full width</label> </div> </li> <li> <a href="https://www.softwareheritage.org">Home</a> </li> <li> <a href="https://forge.softwareheritage.org/">Development</a> </li> <li> <a href="https://docs.softwareheritage.org/devel/">Documentation</a> </li> <li> <a class="swh-donate-link" href="https://www.softwareheritage.org/donate">Donate</a> </li> <li class="swh-position-right"> <a href="{{ status.server_url }}" target="_blank" class="swh-current-status mr-3 d-none d-lg-inline-block d-xl-inline-block"> <span id="swh-current-status-description">Operational</span> <i class="swh-current-status-indicator green"></i> </a> {% url 'logout' as logout_url %} {% if user.is_authenticated %} Logged in as {% if 'OIDC' in user.backend %} <a id="swh-login" href="{% url 'oidc-profile' %}"><strong>{{ user.username }}</strong></a>, <a href= "{% url 'oidc-logout' %}?next_path={% url 'logout' %}?remote_user=1">logout</a> {% else %} <strong id="swh-login">{{ user.username }}</strong>, <a href="{{ logout_url }}">logout</a> {% endif %} {% elif oidc_enabled %} {% if request.path != logout_url %} <a id="swh-login" href="{% url 'oidc-login' %}?next_path={{ request.build_absolute_uri }}">login</a> {% else %} <a id="swh-login" href="{% url 'oidc-login' %}">login</a> {% endif %} {% else %} {% if request.path != logout_url %} <a id="swh-login" href="{% url 'login' %}?next={{ request.build_absolute_uri }}">login</a> {% else %} <a id="swh-login" href="{% url 'login' %}">login</a> {% endif %} {% endif %} </li> </ul> </div> {% if "swh.web.banners" in SWH_DJANGO_APPS %} <div class="swh-banner"> {% include "hiring-banner.html" %} </div> {% endif %} <nav class="main-header navbar navbar-expand-lg navbar-light navbar-static-top swh-navbar {% if 'swh.web.banners' in SWH_DJANGO_APPS %} swh-navbar-banner {% endif %}"> <div class="navbar-header"> <a class="nav-link swh-push-menu" data-widget="pushmenu" data-enable-remember="true" href="#"> <i class="mdi mdi-24px mdi-menu mdi-fw" aria-hidden="true"></i> </a> </div> <div class="navbar" style="width: 94%;"> <div class="swh-navbar-content"> {% block navbar-content %}{% endblock %} {% if request.resolver_match.url_name != 'swh-web-homepage' and request.resolver_match.url_name != 'browse-search' %} <form class="form-horizontal d-none d-md-flex input-group swh-search-navbar needs-validation" id="swh-origins-search-top"> <input class="form-control" placeholder="Enter a SWHID to resolve or keyword(s) to search for in origin URLs" type="text" id="swh-origins-search-top-input" oninput="swh.webapp.validateSWHIDInput(this)" required/> <div class="input-group-append"> <button class="btn btn-primary" type="submit"> <i class="swh-search-icon mdi mdi-24px mdi-magnify" aria-hidden="true"></i> </button> </div> </form> {% endif %} </div> </div> </nav> </div> <aside class="swh-sidebar main-sidebar {% if 'swh.web.banners' in SWH_DJANGO_APPS %} main-sidebar-banner {% endif %} sidebar-no-expand sidebar-light-primary elevation-4 swh-sidebar-{{ sidebar_state }}"> <a href="{% url 'swh-web-homepage' %}" class="brand-link"> <img class="brand-image" src="{% static 'img/swh-logo.png' %}"> <div class="brand-text sitename" href="{% url 'swh-web-homepage' %}"> <span class="first-word">Software</span> <span class="second-word">Heritage</span> </div> </a> <a href="/" class="swh-words-logo"> <div class="swh-words-logo-swh"> <span class="first-word">Software</span> <span class="second-word">Heritage</span> </div> <span>Archive</span> </a> <div class="sidebar"> <nav class="mt-2"> <ul class="nav nav-pills nav-sidebar flex-column" data-widget="treeview" role="menu" data-accordion="false"> <li class="nav-header">Features</li> <li class="nav-item swh-search-item" title="Search archived software"> <a href="{% url 'browse-search' %}" class="nav-link swh-search-link"> <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-magnify"></i> <p>Search</p> </a> </li> - <li class="nav-item swh-vault-item" title="Download archived software from the Vault"> - <a href="{% url 'browse-vault' %}" class="nav-link swh-vault-link"> - <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-download"></i> - <p>Downloads</p> - </a> - </li> + {% if "swh.web.vault" in SWH_DJANGO_APPS %} + <li class="nav-item swh-vault-item" title="Download archived software from the Vault"> + <a href="{% url 'vault' %}" class="nav-link swh-vault-link"> + <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-download"></i> + <p>Downloads</p> + </a> + </li> + {% endif %} {% if "swh.web.save_code_now" in SWH_DJANGO_APPS %} <li class="nav-item swh-origin-save-item" title="Request the saving of a software origin into the archive"> <a href="{% url 'origin-save' %}" class="nav-link swh-origin-save-link"> <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-camera"></i> <p>Save code now</p> </a> </li> {% endif %} {% if "swh.web.add_forge_now" in SWH_DJANGO_APPS %} <li class="nav-item swh-add-forge-now-item" title="Request adding a new forge listing"> <a href="{% url 'forge-add-create' %}" class="nav-link swh-add-forge-now-link"> <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-anvil"></i> <p>Add forge now</p> </a> </li> {% endif %} <li class="nav-item swh-help-item" title="How to browse the archive ?"> <a href="#" class="nav-link swh-help-link" onclick="swh.guided_tour.guidedTourButtonClick(event)"> <i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-help-circle"></i> <p>Help</p> </a> </li> {% if user.is_authenticated %} <li class="nav-header">Administration</li> {% if "swh.web.save_code_now" in SWH_DJANGO_APPS and user.is_staff %} <li class="nav-item swh-origin-save-admin-item" title="Save code now administration"> <a href="{% url 'admin-origin-save-requests' %}" class="nav-link swh-origin-save-admin-link"> <i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-camera"></i> <p>Save code now</p> </a> </li> {% endif %} {% if "swh.web.add_forge_now" in SWH_DJANGO_APPS %} {% if user.is_staff or ADD_FORGE_MODERATOR_PERMISSION in user.get_all_permissions %} <li class="nav-item swh-add-forge-now-moderation-item" title="Add forge now moderation"> <a href="{% url 'add-forge-now-requests-moderation' %}" class="nav-link swh-add-forge-now-moderation-link"> <i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-anvil"></i> <p>Add forge now</p> </a> </li> {% endif %} {% endif %} {% if "swh.web.deposit" in SWH_DJANGO_APPS and user.is_staff or ADMIN_LIST_DEPOSIT_PERMISSION in user.get_all_permissions %} <li class="nav-item swh-deposit-admin-item" title="Deposit administration"> <a href="{% url 'admin-deposit' %}" class="nav-link swh-deposit-admin-link"> <i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-folder-upload"></i> <p>Deposit</p> </a> </li> {% endif %} {% if "swh.web.mailmap" in SWH_DJANGO_APPS and MAILMAP_ADMIN_PERMISSION in user.get_all_permissions %} <li class="nav-item swh-mailmap-admin-item" title="Mailmap administration"> <a href="{% url 'admin-mailmap' %}" class="nav-link swh-mailmap-admin-link"> <i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-email"></i> <p>Mailmap</p> </a> </li> {% endif %} {% endif %} </ul> </nav> </div> </aside> <div class="content-wrapper"> <section class="content"> <div class="container" id="swh-web-content"> {% if swh_web_staging %} <div class="swh-corner-ribbon {% if 'swh.web.banners' in SWH_DJANGO_APPS %} swh-corner-ribbon-banner {% endif %}"> Staging<br/>v{{ swh_web_version }} </div> {% elif swh_web_dev %} <div class="swh-corner-ribbon {% if 'swh.web.banners' in SWH_DJANGO_APPS %} swh-corner-ribbon-banner {% endif %}"> Development<br/>v{{ swh_web_version|split:"+"|first }} </div> {% endif %} {% block content %}{% endblock %} </div> </section> </div> {% include "includes/global-modals.html" %} <footer class="footer"> <div class="container text-center"> <a href="https://www.softwareheritage.org">Software Heritage</a> — Copyright (C) 2015–{% now "Y" %}, The Software Heritage developers. License: <a href="https://www.gnu.org/licenses/agpl.html">GNU AGPLv3+</a>. <br/> The source code of Software Heritage <em>itself</em> is available on our <a href="https://forge.softwareheritage.org/">development forge</a>. <br/> The source code files <em>archived</em> by Software Heritage are available under their own copyright and licenses. <br/> <span class="link-color">Terms of use: </span> <a href="https://www.softwareheritage.org/legal/bulk-access-terms-of-use/">Archive access</a>, <a href="https://www.softwareheritage.org/legal/api-terms-of-use/">API</a>- <a href="https://www.softwareheritage.org/contact/">Contact</a>- {% if "swh.web.jslicenses" in SWH_DJANGO_APPS %} <a href="{% url 'jslicenses' %}" rel="jslicense">JavaScript license information</a>- {% endif %} <a href="{% url 'api-1-homepage' %}">Web API</a><br/> {% if "production" not in DJANGO_SETTINGS_MODULE %} swh-web v{{ swh_web_version }} {% endif %} </div> </footer> <div id="back-to-top"> <a href="#top"><img alt="back to top" src="{% static 'img/arrow-up-small.png' %}" /></a> </div> <script> swh.webapp.setContainerFullWidth(); var statusServerURL = {{ status.server_url|jsonify }}; var statusJsonPath = {{ status.json_path|jsonify }}; swh.webapp.initStatusWidget(statusServerURL + statusJsonPath); </script> </body> </html> diff --git a/swh/web/tests/vault/__init__.py b/swh/web/tests/vault/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/tests/api/views/test_vault.py b/swh/web/tests/vault/test_apiviews.py similarity index 98% rename from swh/web/tests/api/views/test_vault.py rename to swh/web/tests/vault/test_apiviews.py index baf97839..e450b284 100644 --- a/swh/web/tests/api/views/test_vault.py +++ b/swh/web/tests/vault/test_apiviews.py @@ -1,330 +1,330 @@ # Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re import pytest from swh.model.swhids import CoreSWHID from swh.vault.exc import NotFoundExc from swh.web.tests.helpers import ( check_api_get_responses, check_api_post_responses, check_http_get_response, check_http_post_response, ) from swh.web.utils import reverse ##################### # Current API: def test_api_vault_cook(api_client, mocker, directory, revision): - mock_archive = mocker.patch("swh.web.api.views.vault.archive") + mock_archive = mocker.patch("swh.web.vault.api_views.archive") for bundle_type, swhid, content_type, in ( ("flat", f"swh:1:dir:{directory}", "application/gzip"), ("gitfast", f"swh:1:rev:{revision}", "application/gzip"), ("git_bare", f"swh:1:rev:{revision}", "application/x-tar"), ): swhid = CoreSWHID.from_string(swhid) fetch_url = reverse( f"api-1-vault-fetch-{bundle_type.replace('_', '-')}", url_args={"swhid": str(swhid)}, ) stub_cook = { "type": bundle_type, "progress_msg": None, "task_id": 1, "task_status": "done", "swhid": swhid, } stub_fetch = b"content" mock_archive.vault_cook.return_value = stub_cook mock_archive.vault_fetch.return_value = stub_fetch email = "test@test.mail" url = reverse( f"api-1-vault-cook-{bundle_type.replace('_', '-')}", url_args={"swhid": str(swhid)}, query_params={"email": email}, ) rv = check_api_post_responses(api_client, url, data=None, status_code=200) assert rv.data == { "fetch_url": rv.wsgi_request.build_absolute_uri(fetch_url), "progress_message": None, "id": 1, "status": "done", "swhid": str(swhid), } mock_archive.vault_cook.assert_called_with(bundle_type, swhid, email) rv = check_http_get_response(api_client, fetch_url, status_code=200) assert rv["Content-Type"] == content_type assert rv.content == stub_fetch mock_archive.vault_fetch.assert_called_with(bundle_type, swhid) def test_api_vault_cook_notfound( api_client, mocker, directory, revision, unknown_directory, unknown_revision ): mock_vault = mocker.patch("swh.web.utils.archive.vault") mock_vault.cook.side_effect = NotFoundExc("object not found") mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found") mock_vault.progress.side_effect = NotFoundExc("cooking request not found") for bundle_type, swhid in ( ("flat", f"swh:1:dir:{directory}"), ("gitfast", f"swh:1:rev:{revision}"), ("git_bare", f"swh:1:rev:{revision}"), ): swhid = CoreSWHID.from_string(swhid) url = reverse( f"api-1-vault-cook-{bundle_type.replace('_', '-')}", url_args={"swhid": str(swhid)}, ) rv = check_api_get_responses(api_client, url, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"Cooking of {swhid} was never requested." mock_vault.progress.assert_called_with(bundle_type, swhid) for bundle_type, swhid in ( ("flat", f"swh:1:dir:{unknown_directory}"), ("gitfast", f"swh:1:rev:{unknown_revision}"), ("git_bare", f"swh:1:rev:{unknown_revision}"), ): swhid = CoreSWHID.from_string(swhid) url = reverse( f"api-1-vault-cook-{bundle_type.replace('_', '-')}", url_args={"swhid": str(swhid)}, ) rv = check_api_post_responses(api_client, url, data=None, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"{swhid} not found." mock_vault.cook.assert_called_with(bundle_type, swhid, email=None) fetch_url = reverse( f"api-1-vault-fetch-{bundle_type.replace('_', '-')}", url_args={"swhid": str(swhid)}, ) rv = check_api_get_responses(api_client, fetch_url, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"Cooked archive for {swhid} not found." mock_vault.fetch.assert_called_with(bundle_type, swhid) @pytest.mark.parametrize("bundle_type", ["flat", "gitfast", "git_bare"]) def test_api_vault_cook_error_content(api_client, mocker, bundle_type): swhid = "swh:1:cnt:" + "0" * 40 email = "test@test.mail" url = reverse( f"api-1-vault-cook-{bundle_type.replace('_', '-')}", url_args={"swhid": swhid}, query_params={"email": email}, ) rv = check_api_post_responses(api_client, url, data=None, status_code=400) assert rv.data == { "exception": "BadInputExc", "reason": ( "Content objects do not need to be cooked, " "use `/api/1/content/raw/` instead." ), } @pytest.mark.parametrize( "bundle_type,swhid_type,hint", [ ("flat", "rev", True), ("flat", "rel", False), ("flat", "snp", False), ("gitfast", "dir", True), ("gitfast", "rel", False), ("gitfast", "snp", False), ("git_bare", "dir", True), ("git_bare", "rel", False), ("git_bare", "snp", False), ], ) def test_api_vault_cook_error(api_client, mocker, bundle_type, swhid_type, hint): swhid = f"swh:1:{swhid_type}:" + "0" * 40 email = "test@test.mail" url = reverse( f"api-1-vault-cook-{bundle_type.replace('_', '-')}", url_args={"swhid": swhid}, query_params={"email": email}, ) rv = check_api_post_responses(api_client, url, data=None, status_code=400) assert rv.data["exception"] == "BadInputExc" if hint: assert re.match( r"Only .* can be cooked as .* bundles\. Use .*", rv.data["reason"] ) else: assert re.match(r"Only .* can be cooked as .* bundles\.", rv.data["reason"]) ##################### # Legacy API: def test_api_vault_cook_legacy(api_client, mocker, directory, revision): - mock_archive = mocker.patch("swh.web.api.views.vault.archive") + mock_archive = mocker.patch("swh.web.vault.api_views.archive") for obj_type, bundle_type, response_obj_type, obj_id in ( ("directory", "flat", "directory", directory), ("revision_gitfast", "gitfast", "revision", revision), ): swhid = CoreSWHID.from_string(f"swh:1:{obj_type[:3]}:{obj_id}") fetch_url = reverse( f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": str(swhid)}, ) stub_cook = { "type": obj_type, "progress_msg": None, "task_id": 1, "task_status": "done", "swhid": swhid, "obj_type": response_obj_type, "obj_id": obj_id, } stub_fetch = b"content" mock_archive.vault_cook.return_value = stub_cook mock_archive.vault_fetch.return_value = stub_fetch email = "test@test.mail" url = reverse( f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}, query_params={"email": email}, ) rv = check_api_post_responses(api_client, url, data=None, status_code=200) assert rv.data == { "fetch_url": rv.wsgi_request.build_absolute_uri(fetch_url), "progress_message": None, "id": 1, "status": "done", "swhid": str(swhid), "obj_type": response_obj_type, "obj_id": obj_id, } mock_archive.vault_cook.assert_called_with(bundle_type, swhid, email) rv = check_http_get_response(api_client, fetch_url, status_code=200) assert rv["Content-Type"] == "application/gzip" assert rv.content == stub_fetch mock_archive.vault_fetch.assert_called_with(bundle_type, swhid) def test_api_vault_cook_uppercase_hash_legacy(api_client, directory, revision): for obj_type, obj_id in ( ("directory", directory), ("revision_gitfast", revision), ): url = reverse( f"api-1-vault-cook-{obj_type}-uppercase-checksum", url_args={f"{obj_type[:3]}_id": obj_id.upper()}, ) rv = check_http_post_response( api_client, url, data={"email": "test@test.mail"}, status_code=302 ) redirect_url = reverse( f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id} ) assert rv["location"] == redirect_url fetch_url = reverse( f"api-1-vault-fetch-{obj_type}-uppercase-checksum", url_args={f"{obj_type[:3]}_id": obj_id.upper()}, ) rv = check_http_get_response(api_client, fetch_url, status_code=302) redirect_url = reverse( f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}, ) assert rv["location"] == redirect_url def test_api_vault_cook_notfound_legacy( api_client, mocker, directory, revision, unknown_directory, unknown_revision ): mock_vault = mocker.patch("swh.web.utils.archive.vault") mock_vault.cook.side_effect = NotFoundExc("object not found") mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found") mock_vault.progress.side_effect = NotFoundExc("cooking request not found") for obj_type, bundle_type, obj_id in ( ("directory", "flat", directory), ("revision_gitfast", "gitfast", revision), ): url = reverse( f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}, ) swhid = CoreSWHID.from_string(f"swh:1:{obj_type[:3]}:{obj_id}") rv = check_api_get_responses(api_client, url, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"Cooking of {swhid} was never requested." mock_vault.progress.assert_called_with(bundle_type, swhid) for obj_type, bundle_type, obj_id in ( ("directory", "flat", unknown_directory), ("revision_gitfast", "gitfast", unknown_revision), ): swhid = CoreSWHID.from_string(f"swh:1:{obj_type[:3]}:{obj_id}") url = reverse( f"api-1-vault-cook-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id} ) rv = check_api_post_responses(api_client, url, data=None, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"{swhid} not found." mock_vault.cook.assert_called_with(bundle_type, swhid, email=None) fetch_url = reverse( f"api-1-vault-fetch-{obj_type}", url_args={f"{obj_type[:3]}_id": obj_id}, ) # Redirected to the current 'fetch' url rv = check_http_get_response(api_client, fetch_url, status_code=302) redirect_url = reverse( f"api-1-vault-fetch-{bundle_type}", url_args={"swhid": str(swhid)}, ) assert rv["location"] == redirect_url rv = check_api_get_responses(api_client, redirect_url, status_code=404) assert rv.data["exception"] == "NotFoundExc" assert rv.data["reason"] == f"Cooked archive for {swhid} not found." mock_vault.fetch.assert_called_with(bundle_type, swhid) diff --git a/swh/web/tests/vault/test_app.py b/swh/web/tests/vault/test_app.py new file mode 100644 index 00000000..e8f80608 --- /dev/null +++ b/swh/web/tests/vault/test_app.py @@ -0,0 +1,37 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from django.urls import get_resolver + +from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse +from swh.web.vault.urls import urlpatterns + + +@pytest.mark.django_db +def test_banners_deactivate(client, django_settings, directory): + """Check vault feature is deactivated when the swh.web.vault django + application is not in installed apps.""" + + url = reverse("browse-directory", url_args={"sha1_git": directory}) + + resp = check_html_get_response(client, url, status_code=200) + assert_contains(resp, "swh-vault-item") + assert_contains(resp, "swh-vault-download") + + django_settings.SWH_DJANGO_APPS = [ + app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.vault" + ] + + resp = check_html_get_response(client, url, status_code=200) + assert_not_contains(resp, "swh-vault-item") + assert_not_contains(resp, "swh-vault-download") + + vault_view_names = set(urlpattern.name for urlpattern in urlpatterns) + all_view_names = set(get_resolver().reverse_dict.keys()) + assert vault_view_names & all_view_names == set() diff --git a/swh/web/tests/vault/test_views.py b/swh/web/tests/vault/test_views.py new file mode 100644 index 00000000..4a1a4c0b --- /dev/null +++ b/swh/web/tests/vault/test_views.py @@ -0,0 +1,18 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse + + +def test_vault_view(client): + url = reverse("vault") + check_html_get_response(client, url, status_code=200, template_used="vault-ui.html") + + +def test_browse_vault_view(client): + url = reverse("browse-vault") + resp = check_html_get_response(client, url, status_code=302) + assert resp["location"] == reverse("vault") diff --git a/swh/web/vault/__init__.py b/swh/web/vault/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/api/views/vault.py b/swh/web/vault/api_views.py similarity index 100% rename from swh/web/api/views/vault.py rename to swh/web/vault/api_views.py diff --git a/swh/web/browse/templates/includes/vault-common.html b/swh/web/vault/templates/includes/vault-common.html similarity index 100% rename from swh/web/browse/templates/includes/vault-common.html rename to swh/web/vault/templates/includes/vault-common.html diff --git a/swh/web/browse/templates/includes/vault-create-tasks.html b/swh/web/vault/templates/includes/vault-create-tasks.html similarity index 100% rename from swh/web/browse/templates/includes/vault-create-tasks.html rename to swh/web/vault/templates/includes/vault-create-tasks.html diff --git a/swh/web/browse/templates/browse-vault-ui.html b/swh/web/vault/templates/vault-ui.html similarity index 98% rename from swh/web/browse/templates/browse-vault-ui.html rename to swh/web/vault/templates/vault-ui.html index 2c729755..0b0cc7a0 100644 --- a/swh/web/browse/templates/browse-vault-ui.html +++ b/swh/web/vault/templates/vault-ui.html @@ -1,51 +1,51 @@ -{% extends "./browse.html" %} +{% extends "browse.html" %} {% comment %} Copyright (C) 2017-2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% block navbar-content %} <h4>Download archived software</h4> {% endblock %} {% block browse-content %} <p> This interface enables you to track the status of the different Software Heritage Vault cooking tasks created while browsing the archive. </p> <p> Once a cooking task is finished, a link will be made available in order to download the associated archive. </p> <button type="button" class="btn btn-default btn-sm" id="vault-remove-tasks">Remove selected tasks</button> <div class="table-responsive mt-3"> <table class="table swh-table swh-table-striped swh-vault-table" id="vault-cooking-tasks"> <thead> <tr> <th> <div class="custom-control custom-checkbox"> <input type="checkbox" class="custom-control-input" id="vault-tasks-toggle-selection"> <label class="custom-control-label" for="vault-tasks-toggle-selection"></label> </div> </th> <th style="width: 300px">Origin</th> <th style="width: 100px">Bundle type</th> <th>Object info</th> <th style="width: 250px">Cooking status</th> <th style="width: 120px"></th> </tr> </thead> <tbody></tbody> </table> </div> {% include "./includes/vault-common.html" %} <script> swh.webapp.initPage('vault'); swh.vault.initUi(); </script> {% endblock %} diff --git a/swh/web/vault/urls.py b/swh/web/vault/urls.py new file mode 100644 index 00000000..749d7af9 --- /dev/null +++ b/swh/web/vault/urls.py @@ -0,0 +1,30 @@ +# Copyright (C) 2017-2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.http import HttpRequest, HttpResponse +from django.shortcuts import redirect, render +from django.urls import re_path as url + +# register Web API endpoints +import swh.web.vault.api_views # noqa + + +def vault_view(request: HttpRequest) -> HttpResponse: + return render( + request, + "vault-ui.html", + {"heading": "Download archive content from the Vault"}, + ) + + +def browse_vault_view(request: HttpRequest) -> HttpResponse: + return redirect("vault") + + +urlpatterns = [ + url(r"^vault/$", vault_view, name="vault"), + # for backward compatibility + url(r"^browse/vault/$", browse_vault_view, name="browse-vault"), +]