diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js --- a/assets/src/bundles/save/index.js +++ b/assets/src/bundles/save/index.js @@ -7,14 +7,28 @@ import {handleFetchError, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions'; import {swhSpinnerSrc} from 'utils/constants'; +import './save.css'; let saveRequestsTable; -function originSaveRequest(originType, originUrl, - acceptedCallback, pendingCallback, errorCallback) { +function originSaveRequest( + originType, originUrl, extraData, + acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the origin save request let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl); $('.swh-processing-save-request').css('display', 'block'); - fetch(addSaveOriginRequestUrl, {method: 'POST'}) + + let payload = (extraData === {}) ? {method: 'POST'} : { + method: 'POST', + body: JSON.stringify(extraData), + // body: extraData, + headers: { + 'Content-Type': 'application/json' + } + }; + + fetch(addSaveOriginRequestUrl, payload) .then(handleFetchError) .then(response => response.json()) .then(data => { @@ -33,6 +47,15 @@ }); } +export function maybeDisplayExtraInputs() { + // Read the actual selected value and depending on the origin type, display some extra + // inputs or hide them. + const originType = $('#swh-input-visit-type').val(); + const display = originType === 'bundle' ? 'block' : 'none'; + console.log(`originType: ${originType}, display: ${display}`); + $('#optional-origin-forms').css('display', display); +} + export function initOriginSave() { $(document).ready(() => { @@ -45,6 +68,16 @@ for (let originType of data) { $('#swh-input-visit-type').append(``); } + // set git as the default value as before + $('#swh-input-visit-type').val('git'); + + // hard-coding to ease manuel testing, to remove + $('#swh-input-visit-type').val('bundle'); + $('#swh-input-origin-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf'); + $('#swh-input-artifact-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-filename').val('3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-version').val('1.1.4'); + maybeDisplayExtraInputs(); }); saveRequestsTable = $('#swh-origin-save-requests') @@ -189,7 +222,14 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); - originSaveRequest(originType, originUrl, + // read the extra inputs for the bundle type + let extraData = originType !== 'bundle' ? {} : { + 'artifact_url': $('#swh-input-artifact-url').val(), + 'artifact_filename': $('#swh-input-artifact-filename').val(), + 'artifact_version': $('#swh-input-artifact-version').val() + }; + + originSaveRequest(originType, originUrl, extraData, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode, errorData) => { @@ -316,8 +356,9 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); + let extraData = {}; - originSaveRequest(originType, originUrl, + originSaveRequest(originType, originUrl, extraData, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode, errorData) => { diff --git a/assets/src/bundles/save/save.css b/assets/src/bundles/save/save.css new file mode 100644 --- /dev/null +++ b/assets/src/bundles/save/save.css @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2021 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +#optional-origin-forms { + display: none; +} diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -3,6 +3,8 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION @@ -11,6 +13,8 @@ get_save_origin_requests, ) +logger = logging.getLogger(__name__) + @api_route( r"/origin/save/(?P.+)/url/(?P.+)/", @@ -83,11 +87,21 @@ """ + data = request.data or {} if request.method == "POST": bypass_pending_review = request.user.is_authenticated and request.user.has_perm( SWH_AMBASSADOR_PERMISSION ) - sor = create_save_origin_request(visit_type, origin_url, bypass_pending_review) + + if visit_type == "bundle": + logger.error("###### request.data: %s", data) + logger.error("###### artifact url: %s", data.get("artifact_url")) + logger.error("###### artifact filename: %s", data.get("artifact_filename")) + logger.error("###### artifact version: %s", data.get("artifact_version")) + + sor = create_save_origin_request( + visit_type, origin_url, bypass_pending_review, **data + ) del sor["id"] else: sor = get_save_origin_requests(visit_type, origin_url) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -111,7 +111,13 @@ # map visit type to scheduler task # TODO: do not hardcode the task name here (T1157) -_visit_type_task = {"git": "load-git", "hg": "load-hg", "svn": "load-svn"} +_visit_type_task = { + "git": "load-git", + "hg": "load-hg", + "svn": "load-svn", + # TODO: Limit access to ambassador + "bundle": "load-archive-files", +} # map scheduler task status to origin save status @@ -188,14 +194,18 @@ ) -def _check_origin_exists(origin_url: str) -> None: +def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: """Ensure the origin exists, if not raise an explicit message.""" - check = origin_exists(origin_url) - if not check["exists"]: + if not origin_url: + raise BadInputExc("The origin url provided must be set!") + metadata = origin_exists(origin_url) + if not metadata["exists"]: raise BadInputExc( f"The provided origin url ({escape(origin_url)}) does not exist!" ) + return metadata + def _get_visit_info_for_save_request( save_request: SaveOriginRequest, @@ -338,10 +348,9 @@ def create_save_origin_request( - visit_type: str, origin_url: str, bypass_pending_review: bool = False + visit_type: str, origin_url: str, bypass_pending_review: bool = False, **kwargs ) -> SaveOriginRequestInfo: - """ - Create a loading task to save a software origin into the archive. + """Create a loading task to save a software origin into the archive. This function aims to create a software origin loading task trough the use of the swh-scheduler component. @@ -355,8 +364,10 @@ database to keep track of them. Args: - visit_type: the type of visit to perform (e.g git, hg, svn, ...) + visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) origin_url: the url of the origin to save + kwargs: Optional parameters (e.g. artifact_url, artifact_filename, + artifact_version) Raises: BadInputExc: the visit type or origin url is invalid or inexistent @@ -374,10 +385,14 @@ **not created**, **not yet scheduled**, **scheduled**, **succeed** or **failed** - """ _check_visit_type_savable(visit_type) _check_origin_url_valid(origin_url) + + artifact_url = kwargs.get("artifact_url") + if visit_type == "bundle": + metadata = _check_origin_exists(artifact_url) + # if all checks passed so far, we can try and save the origin save_request_status = can_save_origin(origin_url, bypass_pending_review) task = None @@ -386,10 +401,24 @@ # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority - kwargs = { + task_kwargs = { "priority": "high", "url": origin_url, } + if visit_type == "bundle": # some extra arguments are required + assert metadata is not None + task_kwargs = dict( + **task_kwargs, + artifacts=[ + { + "time": metadata["last_modified"], + "url": artifact_url, + "filename": kwargs["artifact_filename"], + "version": kwargs["artifact_version"], + "length": metadata["content_length"], + } + ], + ) sor = None # get list of previously sumitted save requests current_sors = list( @@ -431,7 +460,9 @@ if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict( + _visit_type_task[visit_type], **task_kwargs + ) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,7 +1,7 @@ {% extends "../layout.html" %} {% comment %} -Copyright (C) 2018-2019 The Software Heritage developers +Copyright (C) 2018-2021 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information @@ -33,7 +33,7 @@
- {% comment %} {% endcomment %}
The origin type must be specified
@@ -50,6 +50,21 @@
+
+
+
+ + +
+
+ + +
+
+ + +
+