diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js --- a/assets/src/bundles/save/index.js +++ b/assets/src/bundles/save/index.js @@ -10,11 +10,23 @@ let saveRequestsTable; -function originSaveRequest(originType, originUrl, - acceptedCallback, pendingCallback, errorCallback) { +function originSaveRequest( + originType, originUrl, extraData, + acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the origin save request let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl); $('.swh-processing-save-request').css('display', 'block'); - csrfPost(addSaveOriginRequestUrl) + let headers = {}; + let body = null; + if (extraData !== {}) { + body = JSON.stringify(extraData); + headers = { + 'Content-Type': 'application/json' + }; + }; + + csrfPost(addSaveOriginRequestUrl, headers, body) .then(handleFetchError) .then(response => response.json()) .then(data => { @@ -33,6 +45,14 @@ }); } +export function maybeDisplayExtraInputs() { + // Read the actual selected value and depending on the origin type, display some extra + // inputs or hide them. + const originType = $('#swh-input-visit-type').val(); + const display = originType === 'bundle' ? 'flex' : 'none'; + $('#optional-origin-forms').css('display', display); +} + const userRequestsFilterCheckbox = `
${originType}`); } + // set git as the default value as before + $('#swh-input-visit-type').val('git'); }); saveRequestsTable = $('#swh-origin-save-requests') @@ -220,7 +242,14 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); - originSaveRequest(originType, originUrl, + // read the extra inputs for the bundle type + let extraData = originType !== 'bundle' ? {} : { + 'artifact_url': $('#swh-input-artifact-url').val(), + 'artifact_filename': $('#swh-input-artifact-filename').val(), + 'artifact_version': $('#swh-input-artifact-version').val() + }; + + originSaveRequest(originType, originUrl, extraData, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode, errorData) => { @@ -347,8 +376,9 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); + let extraData = {}; - originSaveRequest(originType, originUrl, + originSaveRequest(originType, originUrl, extraData, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode, errorData) => { diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -19,6 +19,9 @@ 'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.' }; +const anonymousVisitTypes = ['git', 'hg', 'svn']; +const allVisitTypes = ['bundle', 'git', 'hg', 'svn']; + function makeOriginSaveRequest(originType, originUrl) { cy.get('#swh-input-origin-url') .type(originUrl) @@ -472,4 +475,80 @@ }); + it('should list unprivileged visit types when not connected', function() { + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list unprivileged visit types when connected as unprivileged user', function() { + cy.userLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list privileged visit types when connected as ambassador', function() { + cy.ambassadorLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(allVisitTypes); + }); + }); + + it('should display extra inputs when dealing with bundle visit type', function() { + cy.ambassadorLogin(); + cy.visit(url); + + for (let visitType of anonymousVisitTypes) { + cy.get('#swh-input-visit-type').select(visitType); + cy.get('#optional-origin-forms').should('not.be.visible'); + } + + // bundle should display more inputs with the bundle type + cy.get('#swh-input-visit-type').select('bundle'); + cy.get('#optional-origin-forms').should('be.visible'); + + }); + + it('should be allowed to submit bundle save request when connected as ambassador', function() { + let originUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf'; + let artifactUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz'; + let artifactFilename = '3DLDF-1.1.4.tar.gz'; + let artifactVersion = '1.1.4'; + stubSaveRequest({ + requestUrl: this.Urls.api_1_save_origin('bundle', originUrl), + saveRequestStatus: 'accepted', + originUrl: originUrl, + saveTaskStatus: 'not yet scheduled' + }); + + cy.ambassadorLogin(); + cy.visit(url); + + // input new bundle information and submit + cy.get('#swh-input-origin-url') + .type(originUrl) + .get('#swh-input-visit-type') + .select('bundle') + .get('#swh-input-artifact-url') + .type(artifactUrl) + .get('#swh-input-artifact-filename') + .type(artifactFilename) + .get('#swh-input-artifact-version') + .type(artifactVersion) + .get('#swh-save-origin-form') + .submit(); + + cy.wait('@saveRequest').then(() => { + checkAlertVisible('success', saveCodeMsg['success']); + }); + + }); + }); diff --git a/cypress/support/index.js b/cypress/support/index.js --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -53,6 +53,10 @@ return loginUser('user', 'user'); }); +Cypress.Commands.add('ambassadorLogin', () => { + return loginUser('ambassador', 'ambassador'); +}); + before(function() { this.unarchivedRepo = { url: 'https://github.com/SoftwareHeritage/swh-web', diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -83,11 +83,15 @@ """ + data = request.data or {} if request.method == "POST": sor = create_save_origin_request( - visit_type, origin_url, privileged_user(request), user_id=request.user.id + visit_type, + origin_url, + privileged_user(request), + user_id=request.user.id, + **data, ) - del sor["id"] else: sor = get_save_origin_requests(visit_type, origin_url) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -138,8 +138,27 @@ } +def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict: + """Returned the supported task types the user has access to. + + Args: + privileged_user: Flag to determine if all visit types should be returned or not. + Default to False to only list unprivileged visit types. + + Returns: + the dict of supported visit types for the user + + """ + if privileged_user: + task_types = {**_visit_type_task, **_visit_type_task_privileged} + else: + task_types = _visit_type_task + + return task_types + + def get_savable_visit_types(privileged_user: bool = False) -> List[str]: - """Get the list of visit types that can be performed through a save request. + """Return the list of visit types the user can perform save requests on. Args: privileged_user: Flag to determine if all visit types should be returned or not. @@ -149,11 +168,8 @@ the list of saveable visit types """ - task_types = list(_visit_type_task.keys()) - if privileged_user: - task_types += _visit_type_task_privileged.keys() - return sorted(task_types) + return sorted(list(get_savable_visit_types_dict(privileged_user).keys())) def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None: @@ -190,7 +206,13 @@ if exists: size_ = resp.headers.get("Content-Length") content_length = int(size_) if size_ else None - last_modified = resp.headers.get("Last-Modified") + try: + date_str = resp.headers["Last-Modified"] + date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z") + last_modified = date.isoformat() + except (KeyError, ValueError): + # if not provided or not parsable as per the expected format, keep it None + pass return OriginExistenceCheckInfo( origin_url=origin_url, @@ -200,14 +222,18 @@ ) -def _check_origin_exists(origin_url: str) -> None: +def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: """Ensure the origin exists, if not raise an explicit message.""" - check = origin_exists(origin_url) - if not check["exists"]: + if not origin_url: + raise BadInputExc("The origin url provided must be set!") + metadata = origin_exists(origin_url) + if not metadata["exists"]: raise BadInputExc( f"The provided origin url ({escape(origin_url)}) does not exist!" ) + return metadata + def _get_visit_info_for_save_request( save_request: SaveOriginRequest, @@ -354,26 +380,29 @@ origin_url: str, privileged_user: bool = False, user_id: Optional[int] = None, + **kwargs, ) -> SaveOriginRequestInfo: """Create a loading task to save a software origin into the archive. - This function aims to create a software origin loading task - trough the use of the swh-scheduler component. + This function aims to create a software origin loading task trough the use of the + swh-scheduler component. - First, some checks are performed to see if the visit type and origin - url are valid but also if the the save request can be accepted. - If those checks passed, the loading task is then created. - Otherwise, the save request is put in pending or rejected state. + First, some checks are performed to see if the visit type and origin url are valid + but also if the the save request can be accepted. For the 'bundle' visit type, this + also ensures the artifacts actually exists. If those checks passed, the loading task + is then created. Otherwise, the save request is put in pending or rejected state. - All the submitted save requests are logged into the swh-web - database to keep track of them. + All the submitted save requests are logged into the swh-web database to keep track + of them. Args: - visit_type: the type of visit to perform (e.g git, hg, svn, ...) + visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) origin_url: the url of the origin to save - privileged_user: Whether the user has privileged_user access to extra - functionality (e.g. bypass save code now review, access to extra visit type) + privileged: Whether the user has some more privilege than other (bypass + review, access to privileged other visit types) user_id: User identifier (provided when authenticated) + kwargs: Optional parameters (e.g. artifact_url, artifact_filename, + artifact_version) Raises: BadInputExc: the visit type or origin url is invalid or inexistent @@ -392,8 +421,14 @@ **succeed** or **failed** """ + visit_type_tasks = get_savable_visit_types_dict(privileged_user) _check_visit_type_savable(visit_type, privileged_user) _check_origin_url_valid(origin_url) + + artifact_url = kwargs.get("artifact_url") + if visit_type == "bundle": + metadata = _check_origin_exists(artifact_url) + # if all checks passed so far, we can try and save the origin save_request_status = can_save_origin(origin_url, privileged_user) task = None @@ -402,10 +437,25 @@ # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority - kwargs = { + task_kwargs: Dict[str, Any] = { "priority": "high", "url": origin_url, } + if visit_type == "bundle": + # extra arguments for that type are required + assert metadata is not None + task_kwargs = dict( + **task_kwargs, + artifacts=[ + { + "url": artifact_url, + "filename": kwargs["artifact_filename"], + "version": kwargs["artifact_version"], + "time": metadata["last_modified"], + "length": metadata["content_length"], + } + ], + ) sor = None # get list of previously sumitted save requests current_sors = list( @@ -447,7 +497,10 @@ if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict( + visit_type_tasks[visit_type], **task_kwargs + ) + task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted @@ -463,6 +516,7 @@ loading_task_id=task["id"], user_ids=f'"{user_id}"' if user_id else None, ) + # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -255,4 +255,4 @@ content_length: Optional[int] """content length of the artifact""" last_modified: Optional[str] - """Last modification time reported by the server""" + """Last modification time reported by the server (as iso8601 string)""" diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,7 +1,7 @@ {% extends "../layout.html" %} {% comment %} -Copyright (C) 2018-2019 The Software Heritage developers +Copyright (C) 2018-2021 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information @@ -33,7 +33,7 @@
- {% comment %} {% endcomment %}
The origin type must be specified
@@ -50,6 +50,22 @@
+ +