diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js --- a/assets/src/bundles/save/index.js +++ b/assets/src/bundles/save/index.js @@ -7,14 +7,27 @@ import {csrfPost, handleFetchError, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions'; import {swhSpinnerSrc} from 'utils/constants'; +import './save.css'; let saveRequestsTable; -function originSaveRequest(originType, originUrl, - acceptedCallback, pendingCallback, errorCallback) { +function originSaveRequest( + originType, originUrl, extraData, + acceptedCallback, pendingCallback, errorCallback +) { + // Actually trigger the origin save request let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl); $('.swh-processing-save-request').css('display', 'block'); - csrfPost(addSaveOriginRequestUrl) + let headers = {}; + let body = null; + if (extraData !== {}) { + body = JSON.stringify(extraData); + headers = { + 'Content-Type': 'application/json' + }; + }; + + csrfPost(addSaveOriginRequestUrl, headers, body) .then(handleFetchError) .then(response => response.json()) .then(data => { @@ -33,6 +46,15 @@ }); } +export function maybeDisplayExtraInputs() { + // Read the actual selected value and depending on the origin type, display some extra + // inputs or hide them. + const originType = $('#swh-input-visit-type').val(); + const display = originType === 'bundle' ? 'block' : 'none'; + console.log(`originType: ${originType}, display: ${display}`); + $('#optional-origin-forms').css('display', display); +} + export function initOriginSave() { $(document).ready(() => { @@ -45,6 +67,16 @@ for (let originType of data) { $('#swh-input-visit-type').append(``); } + // set git as the default value as before + $('#swh-input-visit-type').val('git'); + + // hard-coding to ease manuel testing, to remove + $('#swh-input-visit-type').val('bundle'); + $('#swh-input-origin-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf'); + $('#swh-input-artifact-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-filename').val('3DLDF-1.1.4.tar.gz'); + $('#swh-input-artifact-version').val('1.1.4'); + maybeDisplayExtraInputs(); }); saveRequestsTable = $('#swh-origin-save-requests') @@ -189,7 +221,14 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); - originSaveRequest(originType, originUrl, + // read the extra inputs for the bundle type + let extraData = originType !== 'bundle' ? {} : { + 'artifact_url': $('#swh-input-artifact-url').val(), + 'artifact_filename': $('#swh-input-artifact-filename').val(), + 'artifact_version': $('#swh-input-artifact-version').val() + }; + + originSaveRequest(originType, originUrl, extraData, () => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert), () => $('#swh-origin-save-request-status').html(saveRequestPendingAlert), (statusCode, errorData) => { @@ -316,8 +355,9 @@ let originType = $('#swh-input-visit-type').val(); let originUrl = $('#swh-input-origin-url').val(); + let extraData = {}; - originSaveRequest(originType, originUrl, + originSaveRequest(originType, originUrl, extraData, () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert), () => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert), (statusCode, errorData) => { diff --git a/assets/src/bundles/save/save.css b/assets/src/bundles/save/save.css new file mode 100644 --- /dev/null +++ b/assets/src/bundles/save/save.css @@ -0,0 +1,10 @@ +/** + * Copyright (C) 2021 The Software Heritage developers + * See the AUTHORS file at the top-level directory of this distribution + * License: GNU Affero General Public License version 3, or any later version + * See top-level LICENSE file for more information + */ + +#optional-origin-forms { + display: none; +} diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py --- a/swh/web/api/views/origin_save.py +++ b/swh/web/api/views/origin_save.py @@ -3,6 +3,8 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging + from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.auth.utils import privileged_user @@ -11,6 +13,8 @@ get_save_origin_requests, ) +logger = logging.getLogger(__name__) + @api_route( r"/origin/save/(?P.+)/url/(?P.+)/", @@ -83,11 +87,22 @@ """ + data = request.data or {} if request.method == "POST": + # TODO: not to be committed, remove + if visit_type == "bundle": + logger.error("###### request.data: %s", data) + logger.error("###### artifact url: %s", data.get("artifact_url")) + logger.error("###### artifact filename: %s", data.get("artifact_filename")) + logger.error("###### artifact version: %s", data.get("artifact_version")) + sor = create_save_origin_request( - visit_type, origin_url, privileged_user(request), user_id=request.user.id + visit_type, + origin_url, + privileged_user(request), + user_id=request.user.id, + **data, ) - del sor["id"] else: sor = get_save_origin_requests(visit_type, origin_url) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -190,7 +190,13 @@ if exists: size_ = resp.headers.get("Content-Length") content_length = int(size_) if size_ else None - last_modified = resp.headers.get("Last-Modified") + try: + date_str = resp.headers["Last-Modified"] + date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z") + last_modified = date.isoformat() + except (KeyError, ValueError): + # if not provided or not parsable, simply keep it None + pass return OriginExistenceCheckInfo( origin_url=origin_url, @@ -200,14 +206,18 @@ ) -def _check_origin_exists(origin_url: str) -> None: +def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo: """Ensure the origin exists, if not raise an explicit message.""" - check = origin_exists(origin_url) - if not check["exists"]: + if not origin_url: + raise BadInputExc("The origin url provided must be set!") + metadata = origin_exists(origin_url) + if not metadata["exists"]: raise BadInputExc( f"The provided origin url ({escape(origin_url)}) does not exist!" ) + return metadata + def _get_visit_info_for_save_request( save_request: SaveOriginRequest, @@ -354,26 +364,29 @@ origin_url: str, privileged_user: bool = False, user_id: Optional[int] = None, + **kwargs, ) -> SaveOriginRequestInfo: """Create a loading task to save a software origin into the archive. - This function aims to create a software origin loading task - trough the use of the swh-scheduler component. + This function aims to create a software origin loading task trough the use of the + swh-scheduler component. - First, some checks are performed to see if the visit type and origin - url are valid but also if the the save request can be accepted. - If those checks passed, the loading task is then created. - Otherwise, the save request is put in pending or rejected state. + First, some checks are performed to see if the visit type and origin url are valid + but also if the the save request can be accepted. For the 'bundle' visit type, this + also ensures the artifacts actually exists. If those checks passed, the loading task + is then created. Otherwise, the save request is put in pending or rejected state. - All the submitted save requests are logged into the swh-web - database to keep track of them. + All the submitted save requests are logged into the swh-web database to keep track + of them. Args: - visit_type: the type of visit to perform (e.g git, hg, svn, ...) + visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...) origin_url: the url of the origin to save - privileged_user: Whether the user has privileged_user access to extra - functionality (e.g. bypass save code now review, access to extra visit type) + privileged: Whether the user has some more privilege than other (bypass + review, access to privileged other visit types) user_id: User identifier (provided when authenticated) + kwargs: Optional parameters (e.g. artifact_url, artifact_filename, + artifact_version) Raises: BadInputExc: the visit type or origin url is invalid or inexistent @@ -394,6 +407,11 @@ """ _check_visit_type_savable(visit_type, privileged_user) _check_origin_url_valid(origin_url) + + artifact_url = kwargs.get("artifact_url") + if visit_type == "bundle": + metadata = _check_origin_exists(artifact_url) + # if all checks passed so far, we can try and save the origin save_request_status = can_save_origin(origin_url, privileged_user) task = None @@ -402,10 +420,25 @@ # task to load it into the archive if save_request_status == SAVE_REQUEST_ACCEPTED: # create a task with high priority - kwargs = { + task_kwargs: Dict[str, Any] = { "priority": "high", "url": origin_url, } + if visit_type == "bundle": + # extra arguments for that type are required + assert metadata is not None + task_kwargs = dict( + **task_kwargs, + artifacts=[ + { + "url": artifact_url, + "filename": kwargs["artifact_filename"], + "version": kwargs["artifact_version"], + "time": metadata["last_modified"], + "length": metadata["content_length"], + } + ], + ) sor = None # get list of previously sumitted save requests current_sors = list( @@ -447,7 +480,9 @@ if can_create_task: # effectively create the scheduler task - task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs) + task_dict = create_oneshot_task_dict( + _visit_type_task[visit_type], **task_kwargs + ) task = scheduler.create_tasks([task_dict])[0] # pending save request has been accepted diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -255,4 +255,4 @@ content_length: Optional[int] """content length of the artifact""" last_modified: Optional[str] - """Last modification time reported by the server""" + """Last modification time reported by the server (as iso8601 string)""" diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html --- a/swh/web/templates/misc/origin-save.html +++ b/swh/web/templates/misc/origin-save.html @@ -1,7 +1,7 @@ {% extends "../layout.html" %} {% comment %} -Copyright (C) 2018-2019 The Software Heritage developers +Copyright (C) 2018-2021 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information @@ -33,7 +33,7 @@
- {% comment %} {% endcomment %}
The origin type must be specified
@@ -50,6 +50,21 @@
+
+
+
+ + +
+
+ + +
+
+ + +
+