diff --git a/assets/src/bundles/add_forge/add-request-history-item.ejs b/assets/src/bundles/add_forge/add-request-history-item.ejs index b17bc1a0..172554c8 100644 --- a/assets/src/bundles/add_forge/add-request-history-item.ejs +++ b/assets/src/bundles/add_forge/add-request-history-item.ejs @@ -1,34 +1,34 @@ <%# Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information %>
-
+

-

-
+
-

<%= event.text %>

+ 
<%= event.text %>
<%if (event.message_source_url !== null) { %>

Open original message in email client

<% } %> <%if (event.new_status !== null) { %>

Status changed to: <%= swh.add_forge.formatRequestStatusName(event.new_status) %>

<% } %>
diff --git a/cypress/integration/add-forge-now-request-dashboard.spec.js b/cypress/integration/add-forge-now-request-dashboard.spec.js index 2acd0772..4c766a2e 100644 --- a/cypress/integration/add-forge-now-request-dashboard.spec.js +++ b/cypress/integration/add-forge-now-request-dashboard.spec.js @@ -1,230 +1,308 @@ /** * Copyright (C) 2022 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ -let requestId, forgeDomain; +let requestId; +let requestForgeDomain; +let requestInboundEmailAddress; + +const requestData = { + forge_type: 'bitbucket', + forge_url: 'test.example.com', + forge_contact_email: 'test@example.com', + forge_contact_name: 'test user', + submitter_forward_username: true, + forge_contact_comment: 'test comment' +}; function createDummyRequest(urls) { cy.task('db:add_forge_now:delete'); cy.userLogin(); - cy.getCookie('csrftoken').its('value').then((token) => { + return cy.getCookie('csrftoken').its('value').then((token) => { cy.request({ method: 'POST', url: urls.api_1_add_forge_request_create(), - body: { - forge_type: 'bitbucket', - forge_url: 'test.example.com', - forge_contact_email: 'test@example.com', - forge_contact_name: 'test user', - submitter_forward_username: true, - forge_contact_comment: 'test comment' - }, + body: requestData, headers: { 'X-CSRFToken': token } }).then((response) => { // setting requestId and forgeDomain from response requestId = response.body.id; - forgeDomain = response.body.forge_domain; + requestForgeDomain = response.body.forge_domain; + requestInboundEmailAddress = response.body.inbound_email_address; // logout the user cy.visit(urls.swh_web_homepage()); cy.contains('a', 'logout').click(); }); }); } +function genEmailSrc() { + return `Message-ID: +Date: Tue, 19 Apr 2022 14:00:56 +0200 +MIME-Version: 1.0 +User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:91.0) Gecko/20100101 + Thunderbird/91.8.0 +To: ${requestData.forge_contact_email} +Cc: ${requestInboundEmailAddress} +Reply-To: ${requestInboundEmailAddress} +Subject: Software Heritage archival request for test.example.com +Content-Language: en-US +From: Test Admin +Content-Type: text/plain; charset=UTF-8; format=flowed +Content-Transfer-Encoding: 7bit + +Dear forge administrator, + +The mission of Software Heritage is to collect, preserve and share all the +publicly available source code (see https://www.softwareheritage.org for more +information). + +We just received a request to add the forge hosted at https://test.example.com to the +list of software origins that are archived, and it is our understanding that you +are the contact person for this forge. + +In order to archive the forge contents, we will have to periodically pull the +public repositories it contains and clone them into the +Software Heritage archive. + +Would you be so kind as to reply to this message to acknowledge the reception +of this email and let us know if there are any special steps we should take in +order to properly archive the public repositories hosted on your infrastructure? + +Thank you in advance for your help. + +Kind regards, +The Software Heritage team +`; +} + describe('Test add forge now request dashboard load', function() { before(function() { // Create an add-forge-request object in the DB createDummyRequest(this.Urls); }); beforeEach(function() { const url = this.Urls.add_forge_now_request_dashboard(requestId); // request dashboard require admin permissions to view cy.adminLogin(); cy.intercept(`${this.Urls.api_1_add_forge_request_get(requestId)}**`).as('forgeRequestGet'); cy.visit(url); }); it('should load add forge request details', function() { cy.wait('@forgeRequestGet'); cy.get('#requestStatus') .should('contain', 'Pending'); cy.get('#requestType') .should('contain', 'bitbucket'); cy.get('#requestURL') .should('contain', 'test.example.com'); cy.get('#requestContactEmail') .should('contain', 'test@example.com'); cy.get('#requestContactName') .should('contain', 'test user'); cy.get('#requestContactEmail') .should('contain', 'test@example.com'); cy.get('#requestContactConsent') .should('contain', 'true'); cy.get('#submitterMessage') .should('contain', 'test comment'); }); it('should show send message link', function() { cy.wait('@forgeRequestGet'); cy.get('#contactForgeAdmin') .should('have.attr', 'emailto') .and('include', 'test@example.com'); cy.get('#contactForgeAdmin') .should('have.attr', 'emailsubject') - .and('include', `Software Heritage archival request for ${forgeDomain}`); + .and('include', `Software Heritage archival request for ${requestForgeDomain}`); }); it('should not show any error message', function() { cy.wait('@forgeRequestGet'); cy.get('#fetchError') .should('have.class', 'd-none'); cy.get('#requestDetails') .should('not.have.class', 'd-none'); }); it('should show error message for an api error', function() { // requesting with a non existing request ID const invalidRequestId = requestId + 10; const url = this.Urls.add_forge_now_request_dashboard(invalidRequestId); cy.intercept(`${this.Urls.api_1_add_forge_request_get(invalidRequestId)}**`, {statusCode: 400}).as('forgeAddInvalidRequest'); cy.visit(url); cy.wait('@forgeAddInvalidRequest'); cy.get('#fetchError') .should('not.have.class', 'd-none'); cy.get('#requestDetails') .should('have.class', 'd-none'); }); it('should load add forge request history', function() { cy.wait('@forgeRequestGet'); cy.get('#requestHistory') .children() .should('have.length', 1); cy.get('#requestHistory') .children() .should('contain', 'New status: Pending'); cy.get('#requestHistory') .should('contain', 'From user (SUBMITTER)'); }); it('should load possible next status', function() { cy.wait('@forgeRequestGet'); // 3 possible next status and the comment option cy.get('#decisionOptions') .children() .should('have.length', 4); }); }); function populateAndSubmitForm() { cy.get('#decisionOptions').select('WAITING_FOR_FEEDBACK'); cy.get('#updateComment').type('This is an update comment'); cy.get('#updateRequestForm').submit(); } -describe('Test forge now request update', function() { +describe('Test add forge now request update', function() { beforeEach(function() { - createDummyRequest(this.Urls); - - const url = this.Urls.add_forge_now_request_dashboard(requestId); - cy.adminLogin(); - // intercept GET API on page load - cy.intercept(`${this.Urls.api_1_add_forge_request_get(requestId)}**`).as('forgeRequestGet'); - // intercept update POST API - cy.intercept('POST', `${this.Urls.api_1_add_forge_request_update(requestId)}**`).as('forgeRequestUpdate'); - cy.visit(url); + createDummyRequest(this.Urls).then(() => { + + this.url = this.Urls.add_forge_now_request_dashboard(requestId); + cy.adminLogin(); + // intercept GET API on page load + cy.intercept(`${this.Urls.api_1_add_forge_request_get(requestId)}**`).as('forgeRequestGet'); + // intercept update POST API + cy.intercept('POST', `${this.Urls.api_1_add_forge_request_update(requestId)}**`).as('forgeRequestUpdate'); + cy.visit(this.url).then(() => { + cy.wait('@forgeRequestGet'); + }); + }); }); it('should submit correct details', function() { - cy.wait('@forgeRequestGet'); populateAndSubmitForm(); // Making sure posting the right data cy.wait('@forgeRequestUpdate').its('request.body') .should('include', 'new_status') .should('include', 'text') .should('include', 'WAITING_FOR_FEEDBACK'); }); it('should show success message', function() { - cy.wait('@forgeRequestGet'); populateAndSubmitForm(); // Making sure showing the success message cy.wait('@forgeRequestUpdate'); cy.get('#userMessage') .should('contain', 'The request status has been updated') .should('not.have.class', 'badge-danger') .should('have.class', 'badge-success'); }); it('should update the dashboard after submit', function() { - cy.wait('@forgeRequestGet'); populateAndSubmitForm(); // Making sure the UI is updated after the submit cy.wait('@forgeRequestGet'); cy.get('#requestStatus') .should('contain', 'Waiting for feedback'); cy.get('#requestHistory') .children() .should('have.length', 2); cy.get('#requestHistory') .children() .should('contain', 'New status: Waiting for feedback'); cy.get('#requestHistory') .children() .should('contain', 'This is an update comment'); cy.get('#requestHistory') .children() .should('contain', 'Status changed to: Waiting for feedback'); cy.get('#decisionOptions') .children() .should('have.length', 2); }); + it('should update the dashboard after receiving email', function() { + const emailSrc = genEmailSrc(); + cy.task('processAddForgeNowInboundEmail', emailSrc); + + // Refresh page and wait for the async request to complete + cy.visit(this.url); + cy.wait('@forgeRequestGet'); + + cy.get('#requestHistory') + .children() + .should('have.length', 2); + + cy.get('#historyItem1') + .click() + .should('contain', 'New status: Waiting for feedback'); + + cy.get('#historyItemBody1') + .find('a') + .should('contain', 'Open original message in email client') + .should('have.prop', 'href').and('contain', '/message-source/').then(function(href) { + cy.request(href).then((response) => { + expect(response.headers['content-type']) + .to.equal('text/email'); + + expect(response.headers['content-disposition']) + .to.match(/filename="add-forge-now-test.example.com-message\d+.eml"/); + + expect(response.body) + .to.equal(emailSrc); + }); + }); + }); + it('should show an error on API failure', function() { cy.intercept('POST', `${this.Urls.api_1_add_forge_request_update(requestId)}**`, {forceNetworkError: true}) .as('updateFailedRequest'); cy.get('#updateComment').type('This is an update comment'); cy.get('#updateRequestForm').submit(); cy.wait('@updateFailedRequest'); cy.get('#userMessage') .should('contain', 'Sorry; Updating the request failed') .should('have.class', 'badge-danger') .should('not.have.class', 'badge-success'); }); }); diff --git a/cypress/plugins/index.js b/cypress/plugins/index.js index fac4b035..a7f7146b 100644 --- a/cypress/plugins/index.js +++ b/cypress/plugins/index.js @@ -1,165 +1,176 @@ /** * Copyright (C) 2019-2022 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ const axios = require('axios'); +const {execFileSync} = require('child_process'); const fs = require('fs'); const sqlite3 = require('sqlite3').verbose(); async function httpGet(url) { const response = await axios.get(url); return response.data; } async function getMetadataForOrigin(originUrl, baseUrl) { const originVisitsApiUrl = `${baseUrl}/api/1/origin/${originUrl}/visits`; const originVisits = await httpGet(originVisitsApiUrl); const lastVisit = originVisits[0]; const snapshotApiUrl = `${baseUrl}/api/1/snapshot/${lastVisit.snapshot}`; const lastOriginSnapshot = await httpGet(snapshotApiUrl); let revision = lastOriginSnapshot.branches.HEAD.target; if (lastOriginSnapshot.branches.HEAD.target_type === 'alias') { revision = lastOriginSnapshot.branches[revision].target; } const revisionApiUrl = `${baseUrl}/api/1/revision/${revision}`; const lastOriginHeadRevision = await httpGet(revisionApiUrl); return { 'directory': lastOriginHeadRevision.directory, 'revision': lastOriginHeadRevision.id, 'snapshot': lastOriginSnapshot.id }; }; function getDatabase() { return new sqlite3.Database('./swh-web-test.sqlite3'); } module.exports = (on, config) => { require('@cypress/code-coverage/task')(on, config); // produce JSON files prior launching browser in order to dynamically generate tests on('before:browser:launch', function(browser, launchOptions) { return new Promise((resolve) => { const p1 = axios.get(`${config.baseUrl}/tests/data/content/code/extensions/`); const p2 = axios.get(`${config.baseUrl}/tests/data/content/code/filenames/`); Promise.all([p1, p2]) .then(function(responses) { fs.writeFileSync('cypress/fixtures/source-file-extensions.json', JSON.stringify(responses[0].data)); fs.writeFileSync('cypress/fixtures/source-file-names.json', JSON.stringify(responses[1].data)); resolve(); }); }); }); on('task', { getSwhTestsData: async() => { if (!global.swhTestsData) { const swhTestsData = {}; swhTestsData.unarchivedRepo = { url: 'https://github.com/SoftwareHeritage/swh-web', type: 'git', revision: '7bf1b2f489f16253527807baead7957ca9e8adde', snapshot: 'd9829223095de4bb529790de8ba4e4813e38672d', rootDirectory: '7d887d96c0047a77e2e8c4ee9bb1528463677663', content: [{ sha1git: 'b203ec39300e5b7e97b6e20986183cbd0b797859' }] }; swhTestsData.origin = [{ url: 'https://github.com/memononen/libtess2', type: 'git', content: [{ path: 'Source/tess.h' }, { path: 'premake4.lua' }], directory: [{ path: 'Source', id: 'cd19126d815470b28919d64b2a8e6a3e37f900dd' }], revisions: [], invalidSubDir: 'Source1' }, { url: 'https://github.com/wcoder/highlightjs-line-numbers.js', type: 'git', content: [{ path: 'src/highlightjs-line-numbers.js' }], directory: [], revisions: ['1c480a4573d2a003fc2630c21c2b25829de49972'], release: { name: 'v2.6.0', id: '6877028d6e5412780517d0bfa81f07f6c51abb41', directory: '5b61d50ef35ca9a4618a3572bde947b8cccf71ad' } }]; for (const origin of swhTestsData.origin) { const metadata = await getMetadataForOrigin(origin.url, config.baseUrl); const directoryApiUrl = `${config.baseUrl}/api/1/directory/${metadata.directory}`; origin.dirContent = await httpGet(directoryApiUrl); origin.rootDirectory = metadata.directory; origin.revisions.push(metadata.revision); origin.snapshot = metadata.snapshot; for (const content of origin.content) { const contentPathApiUrl = `${config.baseUrl}/api/1/directory/${origin.rootDirectory}/${content.path}`; const contentMetaData = await httpGet(contentPathApiUrl); content.name = contentMetaData.name.split('/').slice(-1)[0]; content.sha1git = contentMetaData.target; content.directory = contentMetaData.dir_id; const rawFileUrl = `${config.baseUrl}/browse/content/sha1_git:${content.sha1git}/raw/?filename=${content.name}`; const fileText = await httpGet(rawFileUrl); const fileLines = fileText.split('\n'); content.numberLines = fileLines.length; if (!fileLines[content.numberLines - 1]) { // If last line is empty its not shown content.numberLines -= 1; } } } global.swhTestsData = swhTestsData; } return global.swhTestsData; }, 'db:user_mailmap:delete': () => { const db = getDatabase(); db.serialize(function() { db.run('DELETE FROM user_mailmap'); db.run('DELETE FROM user_mailmap_event'); }); db.close(); return true; }, 'db:user_mailmap:mark_processed': () => { const db = getDatabase(); db.serialize(function() { db.run('UPDATE user_mailmap SET mailmap_last_processing_date=datetime("now", "+1 hour")'); }); db.close(); return true; }, 'db:add_forge_now:delete': () => { const db = getDatabase(); db.serialize(function() { db.run('DELETE FROM add_forge_request_history'); db.run('DELETE FROM sqlite_sequence WHERE name="add_forge_request_history"'); }); db.serialize(function() { db.run('DELETE FROM add_forge_request'); db.run('DELETE FROM sqlite_sequence WHERE name="add_forge_request"'); }); db.close(); return true; + }, + processAddForgeNowInboundEmail(emailSrc) { + try { + execFileSync('django-admin', + ['process_inbound_email', '--settings=swh.web.settings.tests'], + {input: emailSrc}); + return true; + } catch (_) { + return false; + } } }); return config; }; diff --git a/swh/web/add_forge_now/apps.py b/swh/web/add_forge_now/apps.py index 7a27e3d4..76a91ee6 100644 --- a/swh/web/add_forge_now/apps.py +++ b/swh/web/add_forge_now/apps.py @@ -1,13 +1,19 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.apps import AppConfig APP_LABEL = "swh_web_add_forge_now" class AddForgeNowConfig(AppConfig): name = "swh.web.add_forge_now" label = APP_LABEL + + def ready(self): + from ..inbound_email.signals import email_received + from .signal_receivers import handle_inbound_message + + email_received.connect(handle_inbound_message) diff --git a/swh/web/add_forge_now/signal_receivers.py b/swh/web/add_forge_now/signal_receivers.py new file mode 100644 index 00000000..dd38f729 --- /dev/null +++ b/swh/web/add_forge_now/signal_receivers.py @@ -0,0 +1,77 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from email.message import EmailMessage +from typing import Type + +from ..config import get_config +from ..inbound_email.signals import EmailProcessingStatus +from ..inbound_email.utils import get_message_plaintext, get_pks_from_message +from .apps import APP_LABEL +from .models import Request, RequestActorRole, RequestHistory, RequestStatus + + +def handle_inbound_message(sender: Type, **kwargs) -> EmailProcessingStatus: + """Handle inbound email messages for add forge now. + + This uses the from field in the message to set the actor for the new entry in the + request history. We also unconditionally advance the status of requests in the + ``PENDING`` or ``WAITING_FOR_FEEDBACK`` states. + + The message source is saved in the request history as an escape hatch if something + goes wrong during processing. + + """ + + message = kwargs["message"] + assert isinstance(message, EmailMessage) + + base_address = get_config()["add_forge_now"]["email_address"] + + pks = get_pks_from_message( + salt=APP_LABEL, base_address=base_address, message=message + ) + + if not pks: + return EmailProcessingStatus.IGNORED + + for pk in pks: + try: + request = Request.objects.get(pk=pk) + except Request.DoesNotExist: + continue + + request_updated = False + + message_plaintext = get_message_plaintext(message) + if message_plaintext: + history_text = message_plaintext.decode("utf-8", errors="replace") + else: + history_text = ( + "Could not parse the message contents, see the original message." + ) + + history_entry = RequestHistory( + request=request, + actor=str(message["from"]), + actor_role=RequestActorRole.EMAIL.name, + text=history_text, + message_source=bytes(message), + ) + + new_status = { + RequestStatus.PENDING: RequestStatus.WAITING_FOR_FEEDBACK, + RequestStatus.WAITING_FOR_FEEDBACK: RequestStatus.FEEDBACK_TO_HANDLE, + }.get(RequestStatus[request.status]) + + if new_status: + request.status = history_entry.new_status = new_status.name + request_updated = True + + history_entry.save() + if request_updated: + request.save() + + return EmailProcessingStatus.PROCESSED