diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js --- a/assets/src/bundles/save/index.js +++ b/assets/src/bundles/save/index.js @@ -10,11 +10,23 @@ let saveRequestsTable; -function originSaveRequest(originType, originUrl, - acceptedCallback, pendingCallback, errorCallback) { +function originSaveRequest( + originType, originUrl, extraData, + acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the origin save request let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl); $('.swh-processing-save-request').css('display', 'block'); - csrfPost(addSaveOriginRequestUrl) + let headers = {}; + let body = null; + if (extraData !== {}) { + body = JSON.stringify(extraData); + headers = { + 'Content-Type': 'application/json' + }; + }; + + csrfPost(addSaveOriginRequestUrl, headers, body) .then(handleFetchError) .then(response => response.json()) .then(data => { @@ -33,6 +45,14 @@ }); } +export function maybeDisplayExtraInputs() { + // Read the actual selected value and depending on the origin type, display some extra + // inputs or hide them. + const originType = $('#swh-input-visit-type').val(); + const display = originType === 'bundle' ? 'flex' : 'none'; + $('#optional-origin-forms').css('display', display); +} + const userRequestsFilterCheckbox = `
${originType}`); } + // set git as the default value as before + $('#swh-input-visit-type').val('git'); }); saveRequestsTable = $('#swh-origin-save-requests') @@ -220,7 +242,14 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); - originSaveRequest(originType, originUrl, + // read the extra inputs for the bundle type + let extraData = originType !== 'bundle' ? {} : { + 'artifact_url': $('#swh-input-artifact-url').val(), + 'artifact_filename': $('#swh-input-artifact-filename').val(), + 'artifact_version': $('#swh-input-artifact-version').val() + }; + + originSaveRequest(originType, originUrl, extraData, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode, errorData) => { @@ -347,8 +376,9 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); + let extraData = {}; - originSaveRequest(originType, originUrl, + originSaveRequest(originType, originUrl, extraData, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode, errorData) => { diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -19,6 +19,9 @@ 'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.' }; +const anonymousVisitTypes = ['git', 'hg', 'svn']; +const allVisitTypes = ['bundle', 'git', 'hg', 'svn']; + function makeOriginSaveRequest(originType, originUrl) { cy.get('#swh-input-origin-url') .type(originUrl) @@ -472,4 +475,80 @@ }); + it('should list unprivileged visit types when not connected', function() { + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list unprivileged visit types when connected as unprivileged user', function() { + cy.userLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list privileged visit types when connected as ambassador', function() { + cy.ambassadorLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(allVisitTypes); + }); + }); + + it('should display extra inputs when dealing with bundle visit type', function() { + cy.ambassadorLogin(); + cy.visit(url); + + for (let visitType of anonymousVisitTypes) { + cy.get('#swh-input-visit-type').select(visitType); + cy.get('#optional-origin-forms').should('not.be.visible'); + } + + // bundle should display more inputs with the bundle type + cy.get('#swh-input-visit-type').select('bundle'); + cy.get('#optional-origin-forms').should('be.visible'); + + }); + + it('should be allowed to submit bundle save request when connected as ambassador', function() { + let originUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf'; + let artifactUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz'; + let artifactFilename = '3DLDF-1.1.4.tar.gz'; + let artifactVersion = '1.1.4'; + stubSaveRequest({ + requestUrl: this.Urls.api_1_save_origin('bundle', originUrl), + saveRequestStatus: 'accepted', + originUrl: originUrl, + saveTaskStatus: 'not yet scheduled' + }); + + cy.ambassadorLogin(); + cy.visit(url); + + // input new bundle information and submit + cy.get('#swh-input-origin-url') + .type(originUrl) + .get('#swh-input-visit-type') + .select('bundle') + .get('#swh-input-artifact-url') + .type(artifactUrl) + .get('#swh-input-artifact-filename') + .type(artifactFilename) + .get('#swh-input-artifact-version') + .type(artifactVersion) + .get('#swh-save-origin-form') + .submit(); + + cy.wait('@saveRequest').then(() => { + checkAlertVisible('success', saveCodeMsg['success']); + }); + + }); + }); diff --git a/cypress/support/index.js b/cypress/support/index.js --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -53,6 +53,10 @@ return loginUser('user', 'user'); }); +Cypress.Commands.add('ambassadorLogin', () => { + return loginUser('ambassador', 'ambassador'); +}); + before(function() { this.unarchivedRepo = { url: 'https://github.com/SoftwareHeritage/swh-web', diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -5,7 +5,7 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route -from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION +from swh.web.auth.utils import privileged_user from swh.web.common.origin_save import ( create_save_origin_request, get_save_origin_requests, @@ -83,12 +83,14 @@ """ + data = request.data or {} if request.method == "POST": - bypass_pending_review = request.user.is_authenticated and request.user.has_perm( - SWH_AMBASSADOR_PERMISSION - ) sor = create_save_origin_request( - visit_type, origin_url, bypass_pending_review, user_id=request.user.id + visit_type, + origin_url, + privileged_user(request), + user_id=request.user.id, + **data, ) del sor["id"] else: diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py --- a/swh/web/auth/utils.py +++ b/swh/web/auth/utils.py @@ -70,3 +70,14 @@ The decrypted data """ return _get_fernet(password, salt).decrypt(data) + + +def privileged_user(request) -> bool: + """Determine whether a user is authenticated and is a privileged one (e.g ambassador). + This allows such user to have access to some more actions (e.g. bypass save code now + review, access to 'bundle' type...) + + """ + return request.user.is_authenticated and request.user.has_perm( + SWH_AMBASSADOR_PERMISSION + ) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -113,6 +113,10 @@ # TODO: do not hardcode the task name here (T1157) _visit_type_task = {"git": "load-git", "hg": "load-hg", "svn": "load-svn"} +_visit_type_task_privileged = { + "bundle": "load-archive-files", +} + # map scheduler task status to origin save status _save_task_status = { @@ -134,23 +138,47 @@ } -def get_savable_visit_types() -> List[str]: +def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict: + """Returned the supported task types the user has access to. + + Args: + privileged_user: Flag to determine if all visit types should be returned or not. + Default to False to only list unprivileged visit types. + + Returns: + the dict of supported visit types for the user + """ - Get the list of visit types that can be performed - through a save request. + if privileged_user: + task_types = {**_visit_type_task, **_visit_type_task_privileged} + else: + task_types = _visit_type_task + + return task_types + + +def get_savable_visit_types(privileged_user: bool = False) -> List[str]: + """Return the list of visit types the user can perform save requests on. + + Args: + privileged_user: Flag to determine if all visit types should be returned or not. + Default to False to only list unprivileged visit types. Returns: - list: the list of saveable visit types + the list of saveable visit types + """ - return sorted(list(_visit_type_task.keys())) + return sorted(list(get_savable_visit_types_dict(privileged_user).keys())) -def _check_visit_type_savable(visit_type: str) -> None: - allowed_visit_types = ", ".join(get_savable_visit_types()) - if visit_type not in _visit_type_task: + +def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None: + visit_type_tasks = get_savable_visit_types(privileged_user) + if visit_type not in visit_type_tasks: + allowed_visit_types = ", ".join(visit_type_tasks) raise BadInputExc( - "Visit of type %s can not be saved! " - "Allowed types are the following: %s" % (visit_type, allowed_visit_types) + f"Visit of type {visit_type} can not be saved! " + f"Allowed types are the following: {allowed_visit_types}" ) @@ -178,7 +206,13 @@ if exists: size_ = resp.headers.get("Content-Length") content_length = int(size_) if size_ else None - last_modified = resp.headers.get("Last-Modified") + try: + date_str = resp.headers["Last-Modified"] + date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z") + last_modified = date.isoformat() + except (KeyError, ValueError): + # if not provided or not parsable as per the expected format, keep it None + pass return OriginExistenceCheckInfo( origin_url=origin_url, @@ -188,14 +222,18 @@ ) -def _check_origin_exists(origin_url: str) -> None: +def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: """Ensure the origin exists, if not raise an explicit message.""" - check = origin_exists(origin_url) - if not check["exists"]: + if not origin_url: + raise BadInputExc("The origin url provided must be set!") + metadata = origin_exists(origin_url) + if not metadata["exists"]: raise BadInputExc( f"The provided origin url ({escape(origin_url)}) does not exist!" ) + return metadata + def _get_visit_info_for_save_request( save_request: SaveOriginRequest, @@ -340,26 +378,31 @@ def create_save_origin_request( visit_type: str, origin_url: str, - bypass_pending_review: bool = False, + privileged_user: bool = False, user_id: Optional[int] = None, + **kwargs, ) -> SaveOriginRequestInfo: - """ - Create a loading task to save a software origin into the archive. + """Create a loading task to save a software origin into the archive. - This function aims to create a software origin loading task - trough the use of the swh-scheduler component. + This function aims to create a software origin loading task trough the use of the + swh-scheduler component. - First, some checks are performed to see if the visit type and origin - url are valid but also if the the save request can be accepted. - If those checks passed, the loading task is then created. - Otherwise, the save request is put in pending or rejected state. + First, some checks are performed to see if the visit type and origin url are valid + but also if the the save request can be accepted. For the 'bundle' visit type, this + also ensures the artifacts actually exists. If those checks passed, the loading task + is then created. Otherwise, the save request is put in pending or rejected state. - All the submitted save requests are logged into the swh-web - database to keep track of them. + All the submitted save requests are logged into the swh-web database to keep track + of them. Args: - visit_type: the type of visit to perform (e.g git, hg, svn, ...) + visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) origin_url: the url of the origin to save + privileged: Whether the user has some more privilege than other (bypass + review, access to privileged other visit types) + user_id: User identifier (provided when authenticated) + kwargs: Optional parameters (e.g. artifact_url, artifact_filename, + artifact_version) Raises: BadInputExc: the visit type or origin url is invalid or inexistent @@ -377,22 +420,42 @@ **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** - """ - _check_visit_type_savable(visit_type) + visit_type_tasks = get_savable_visit_types_dict(privileged_user) + _check_visit_type_savable(visit_type, privileged_user) _check_origin_url_valid(origin_url) + + artifact_url = kwargs.get("artifact_url") + if visit_type == "bundle": + metadata = _check_origin_exists(artifact_url) + # if all checks passed so far, we can try and save the origin - save_request_status = can_save_origin(origin_url, bypass_pending_review) + save_request_status = can_save_origin(origin_url, privileged_user) task = None # if the origin save request is accepted, create a scheduler # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority - kwargs = { + task_kwargs: Dict[str, Any] = { "priority": "high", "url": origin_url, } + if visit_type == "bundle": + # extra arguments for that type are required + assert metadata is not None + task_kwargs = dict( + **task_kwargs, + artifacts=[ + { + "url": artifact_url, + "filename": kwargs["artifact_filename"], + "version": kwargs["artifact_version"], + "time": metadata["last_modified"], + "length": metadata["content_length"], + } + ], + ) sor = None # get list of previously sumitted save requests current_sors = list( @@ -434,7 +497,10 @@ if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict( + visit_type_tasks[visit_type], **task_kwargs + ) + task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted @@ -450,6 +516,7 @@ loading_task_id=task["id"], user_ids=f'"{user_id}"' if user_id else None, ) + # save request must be manually reviewed for acceptation elif save_request_status == SAVE_REQUEST_PENDING: # check if there is already such a save request already submitted, @@ -778,7 +845,8 @@ SAVE_TASK_RUNNING, ) - visit_types = get_savable_visit_types() + # for metrics, we want access to all visit types + visit_types = get_savable_visit_types(privileged_user=True) labels_set = product(request_statuses, visit_types) diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -1,4 +1,4 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -125,15 +125,16 @@ @register.filter -def visit_type_savable(visit_type): +def visit_type_savable(visit_type: str) -> bool: """Django template filter to check if a save request can be created for a given visit type. Args: - visit_type (str): the type of visit + visit_type: the type of visit Returns: If the visit type is saveable or not + """ return visit_type in get_savable_visit_types() diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -255,4 +255,4 @@ content_length: Optional[int] """content length of the artifact""" last_modified: Optional[str] - """Last modification time reported by the server""" + """Last modification time reported by the server (as iso8601 string)""" diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py --- a/swh/web/misc/origin_save.py +++ b/swh/web/misc/origin_save.py @@ -9,6 +9,7 @@ from django.http import JsonResponse from django.shortcuts import render +from swh.web.auth.utils import privileged_user from swh.web.common.models import SaveOriginRequest from swh.web.common.origin_save import ( get_savable_visit_types, @@ -24,8 +25,11 @@ ) -def _visit_save_types_list(request): - visit_types = get_savable_visit_types() +def _visit_save_types_list(request) -> JsonResponse: + """Return the list of supported visit types as json response + + """ + visit_types = get_savable_visit_types(privileged_user(request)) return JsonResponse(visit_types, safe=False) diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,7 +1,7 @@ {% extends "../layout.html" %} {% comment %} -Copyright (C) 2018-2019 The Software Heritage developers +Copyright (C) 2018-2021 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information @@ -33,7 +33,7 @@
- {% comment %} {% endcomment %}
The origin type must be specified
@@ -50,6 +50,22 @@
+ +