diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js --- a/assets/src/bundles/save/index.js +++ b/assets/src/bundles/save/index.js @@ -7,14 +7,27 @@ import {csrfPost, handleFetchError, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions'; import {swhSpinnerSrc} from 'utils/constants'; +import './save.css'; let saveRequestsTable; -function originSaveRequest(originType, originUrl, - acceptedCallback, pendingCallback, errorCallback) { +function originSaveRequest( + originType, originUrl, extraData, + acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the origin save request let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl); $('.swh-processing-save-request').css('display', 'block'); - csrfPost(addSaveOriginRequestUrl) + let headers = {}; + let body = null; + if (extraData !== {}) { + body = JSON.stringify(extraData); + headers = { + 'Content-Type': 'application/json' + }; + }; + + csrfPost(addSaveOriginRequestUrl, headers, body) .then(handleFetchError) .then(response => response.json()) .then(data => { @@ -33,6 +46,14 @@ }); } +export function maybeDisplayExtraInputs() { + // Read the actual selected value and depending on the origin type, display some extra + // inputs or hide them. + const originType = $('#swh-input-visit-type').val(); + const display = originType === 'bundle' ? 'block' : 'none'; + $('#optional-origin-forms').css('display', display); +} + const userRequestsFilterCheckbox = `
${originType}`); } + // set git as the default value as before + $('#swh-input-visit-type').val('git'); }); saveRequestsTable = $('#swh-origin-save-requests') @@ -220,7 +243,14 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); - originSaveRequest(originType, originUrl, + // read the extra inputs for the bundle type + let extraData = originType !== 'bundle' ? {} : { + 'artifact_url': $('#swh-input-artifact-url').val(), + 'artifact_filename': $('#swh-input-artifact-filename').val(), + 'artifact_version': $('#swh-input-artifact-version').val() + }; + + originSaveRequest(originType, originUrl, extraData, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode, errorData) => { @@ -347,8 +377,9 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); + let extraData = {}; - originSaveRequest(originType, originUrl, + originSaveRequest(originType, originUrl, extraData, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode, errorData) => { diff --git a/assets/src/bundles/save/save.css b/assets/src/bundles/save/save.css new file mode 100644 --- /dev/null +++ b/assets/src/bundles/save/save.css @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2021 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +#optional-origin-forms { + display: none; +} diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js --- a/cypress/integration/origin-save.spec.js +++ b/cypress/integration/origin-save.spec.js @@ -19,6 +19,9 @@ 'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.' }; +const anonymousVisitTypes = ['git', 'hg', 'svn']; +const allVisitTypes = ['bundle', 'git', 'hg', 'svn']; + function makeOriginSaveRequest(originType, originUrl) { cy.get('#swh-input-origin-url') .type(originUrl) @@ -472,4 +475,43 @@ }); + it('should list unprivileged visit types when not connected', function() { + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list unprivileged visit types when connected as unprivileged user', function() { + cy.userLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(anonymousVisitTypes); + }); + }); + + it('should list privileged visit types when connected as ambassador', function() { + cy.ambassadorLogin(); + cy.visit(url); + cy.get('#swh-input-visit-type').children('option').then(options => { + const actual = [...options].map(o => o.value); + expect(actual).to.deep.eq(allVisitTypes); + }); + }); + + // TODO + it('should be able to submit bundle visit when connected as ambassador', function() { + // check it's 'hidden' when not 'bundle' type selected + // select bundle, check it's displayed otherwise + // $('#optional-origin-forms').css('display', display); + // input some bundle and simulate a save code now + $('#swh-input-visit-type').val('bundle'); + $('#swh-input-origin-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf'); + $('#swh-input-artifact-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-filename').val('3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-version').val('1.1.4'); + }); + }); diff --git a/cypress/support/index.js b/cypress/support/index.js --- a/cypress/support/index.js +++ b/cypress/support/index.js @@ -53,6 +53,10 @@ return loginUser('user', 'user'); }); +Cypress.Commands.add('ambassadorLogin', () => { + return loginUser('ambassador', 'ambassador'); +}); + before(function() { this.unarchivedRepo = { url: 'https://github.com/SoftwareHeritage/swh-web', diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -3,6 +3,8 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.auth.utils import privileged_user @@ -11,6 +13,8 @@ get_save_origin_requests, ) +logger = logging.getLogger(__name__) + @api_route( r"/origin/save/(?P.+)/url/(?P.+)/", @@ -83,11 +87,15 @@ """ + data = request.data or {} if request.method == "POST": sor = create_save_origin_request( - visit_type, origin_url, privileged_user(request), user_id=request.user.id + visit_type, + origin_url, + privileged_user(request), + user_id=request.user.id, + **data, ) - del sor["id"] else: sor = get_save_origin_requests(visit_type, origin_url) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -138,8 +138,27 @@ } +def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict: + """Returned the supported task types the user has access to. + + Args: + privileged_user: Flag to determine if all visit types should be returned or not. + Default to False to only list unprivileged visit types. + + Returns: + the dict of supported visit types for the user + + """ + if privileged_user: + task_types = {**_visit_type_task, **_visit_type_task_privileged} + else: + task_types = _visit_type_task + + return task_types + + def get_savable_visit_types(privileged_user: bool = False) -> List[str]: - """Get the list of visit types that can be performed through a save request. + """Return the list of visit types the user can perform save requests on. Args: privileged_user: Flag to determine if all visit types should be returned or not. @@ -149,11 +168,8 @@ the list of saveable visit types """ - task_types = list(_visit_type_task.keys()) - if privileged_user: - task_types += _visit_type_task_privileged.keys() - return sorted(task_types) + return sorted(list(get_savable_visit_types_dict(privileged_user).keys())) def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None: @@ -190,7 +206,13 @@ if exists: size_ = resp.headers.get("Content-Length") content_length = int(size_) if size_ else None - last_modified = resp.headers.get("Last-Modified") + try: + date_str = resp.headers["Last-Modified"] + date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z") + last_modified = date.isoformat() + except (KeyError, ValueError): + # if not provided or not parsable, simply keep it None + pass return OriginExistenceCheckInfo( origin_url=origin_url, @@ -200,14 +222,18 @@ ) -def _check_origin_exists(origin_url: str) -> None: +def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: """Ensure the origin exists, if not raise an explicit message.""" - check = origin_exists(origin_url) - if not check["exists"]: + if not origin_url: + raise BadInputExc("The origin url provided must be set!") + metadata = origin_exists(origin_url) + if not metadata["exists"]: raise BadInputExc( f"The provided origin url ({escape(origin_url)}) does not exist!" ) + return metadata + def _get_visit_info_for_save_request( save_request: SaveOriginRequest, @@ -354,26 +380,29 @@ origin_url: str, privileged_user: bool = False, user_id: Optional[int] = None, + **kwargs, ) -> SaveOriginRequestInfo: """Create a loading task to save a software origin into the archive. - This function aims to create a software origin loading task - trough the use of the swh-scheduler component. + This function aims to create a software origin loading task trough the use of the + swh-scheduler component. - First, some checks are performed to see if the visit type and origin - url are valid but also if the the save request can be accepted. - If those checks passed, the loading task is then created. - Otherwise, the save request is put in pending or rejected state. + First, some checks are performed to see if the visit type and origin url are valid + but also if the the save request can be accepted. For the 'bundle' visit type, this + also ensures the artifacts actually exists. If those checks passed, the loading task + is then created. Otherwise, the save request is put in pending or rejected state. - All the submitted save requests are logged into the swh-web - database to keep track of them. + All the submitted save requests are logged into the swh-web database to keep track + of them. Args: - visit_type: the type of visit to perform (e.g git, hg, svn, ...) + visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) origin_url: the url of the origin to save - privileged_user: Whether the user has privileged_user access to extra - functionality (e.g. bypass save code now review, access to extra visit type) + privileged: Whether the user has some more privilege than other (bypass + review, access to privileged other visit types) user_id: User identifier (provided when authenticated) + kwargs: Optional parameters (e.g. artifact_url, artifact_filename, + artifact_version) Raises: BadInputExc: the visit type or origin url is invalid or inexistent @@ -392,8 +421,14 @@ **succeed** or **failed** """ + visit_type_tasks = get_savable_visit_types_dict(privileged_user) _check_visit_type_savable(visit_type, privileged_user) _check_origin_url_valid(origin_url) + + artifact_url = kwargs.get("artifact_url") + if visit_type == "bundle": + metadata = _check_origin_exists(artifact_url) + # if all checks passed so far, we can try and save the origin save_request_status = can_save_origin(origin_url, privileged_user) task = None @@ -402,10 +437,25 @@ # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority - kwargs = { + task_kwargs: Dict[str, Any] = { "priority": "high", "url": origin_url, } + if visit_type == "bundle": + # extra arguments for that type are required + assert metadata is not None + task_kwargs = dict( + **task_kwargs, + artifacts=[ + { + "url": artifact_url, + "filename": kwargs["artifact_filename"], + "version": kwargs["artifact_version"], + "time": metadata["last_modified"], + "length": metadata["content_length"], + } + ], + ) sor = None # get list of previously sumitted save requests current_sors = list( @@ -447,7 +497,9 @@ if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict( + visit_type_tasks[visit_type], **task_kwargs + ) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -255,4 +255,4 @@ content_length: Optional[int] """content length of the artifact""" last_modified: Optional[str] - """Last modification time reported by the server""" + """Last modification time reported by the server (as iso8601 string)""" diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,7 +1,7 @@ {% extends "../layout.html" %} {% comment %} -Copyright (C) 2018-2019 The Software Heritage developers +Copyright (C) 2018-2021 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information @@ -33,7 +33,7 @@
- {% comment %} {% endcomment %}
The origin type must be specified
@@ -50,6 +50,21 @@
+
+
+
+ + +
+
+ + +
+
+ + +
+