Page MenuHomeSoftware Heritage
Files F7123692

D5676.id20324.diff
No OneTemporary
Actions

D5676.id20324.diff
View Options

diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js
--- a/assets/src/bundles/save/index.js
+++ b/assets/src/bundles/save/index.js
@@ -7,14 +7,27 @@
import {csrfPost, handleFetchError, isGitRepoUrl, htmlAlert, removeUrlFragment} from 'utils/functions';
import {swhSpinnerSrc} from 'utils/constants';
+import './save.css';
let saveRequestsTable;
-function originSaveRequest(originType, originUrl,
- acceptedCallback, pendingCallback, errorCallback) {
+function originSaveRequest(
+ originType, originUrl, extraData,
+ acceptedCallback, pendingCallback, errorCallback
+) {
+ // Actually trigger the origin save request
let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl);
$('.swh-processing-save-request').css('display', 'block');
- csrfPost(addSaveOriginRequestUrl)
+ let headers = {};
+ let body = null;
+ if (extraData !== {}) {
+ body = JSON.stringify(extraData);
+ headers = {
+ 'Content-Type': 'application/json'
+ };
+ };
+
+ csrfPost(addSaveOriginRequestUrl, headers, body)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
@@ -33,6 +46,15 @@
});
}
+export function maybeDisplayExtraInputs() {
+ // Read the actual selected value and depending on the origin type, display some extra
+ // inputs or hide them.
+ const originType = $('#swh-input-visit-type').val();
+ const display = originType === 'bundle' ? 'block' : 'none';
+ console.log(`originType: ${originType}, display: ${display}`);
+ $('#optional-origin-forms').css('display', display);
+}
+
export function initOriginSave() {
$(document).ready(() => {
@@ -45,6 +67,16 @@
for (let originType of data) {
$('#swh-input-visit-type').append(`<option value="${originType}">${originType}</option>`);
}
+ // set git as the default value as before
+ $('#swh-input-visit-type').val('git');
+
+ // hard-coding to ease manuel testing, to remove
+ $('#swh-input-visit-type').val('bundle');
+ $('#swh-input-origin-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf');
+ $('#swh-input-artifact-url').val('https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz');
+ $('#swh-input-artifact-filename').val('3DLDF-1.1.4.tar.gz');
+ $('#swh-input-artifact-version').val('1.1.4');
+ maybeDisplayExtraInputs();
});
saveRequestsTable = $('#swh-origin-save-requests')
@@ -189,7 +221,14 @@
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
- originSaveRequest(originType, originUrl,
+ // read the extra inputs for the bundle type
+ let extraData = originType !== 'bundle' ? {} : {
+ 'artifact_url': $('#swh-input-artifact-url').val(),
+ 'artifact_filename': $('#swh-input-artifact-filename').val(),
+ 'artifact_version': $('#swh-input-artifact-version').val()
+ };
+
+ originSaveRequest(originType, originUrl, extraData,
() => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert),
() => $('#swh-origin-save-request-status').html(saveRequestPendingAlert),
(statusCode, errorData) => {
@@ -316,8 +355,9 @@
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
+ let extraData = {};
- originSaveRequest(originType, originUrl,
+ originSaveRequest(originType, originUrl, extraData,
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert),
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert),
(statusCode, errorData) => {
diff --git a/assets/src/bundles/save/save.css b/assets/src/bundles/save/save.css
new file mode 100644
--- /dev/null
+++ b/assets/src/bundles/save/save.css
@@ -0,0 +1,10 @@
+/**
+ * Copyright (C) 2021 The Software Heritage developers
+ * See the AUTHORS file at the top-level directory of this distribution
+ * License: GNU Affero General Public License version 3, or any later version
+ * See top-level LICENSE file for more information
+ */
+
+#optional-origin-forms {
+ display: none;
+}
diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py
--- a/swh/web/api/views/origin_save.py
+++ b/swh/web/api/views/origin_save.py
@@ -3,6 +3,8 @@
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import logging
+
from swh.web.api.apidoc import api_doc, format_docstring
from swh.web.api.apiurls import api_route
from swh.web.auth.utils import privileged_user
@@ -11,6 +13,8 @@
get_save_origin_requests,
)
+logger = logging.getLogger(__name__)
+
@api_route(
r"/origin/save/(?P<visit_type>.+)/url/(?P<origin_url>.+)/",
@@ -83,11 +87,22 @@
"""
+ data = request.data or {}
if request.method == "POST":
+ # TODO: not to be committed, remove
+ if visit_type == "bundle":
+ logger.error("###### request.data: %s", data)
+ logger.error("###### artifact url: %s", data.get("artifact_url"))
+ logger.error("###### artifact filename: %s", data.get("artifact_filename"))
+ logger.error("###### artifact version: %s", data.get("artifact_version"))
+
sor = create_save_origin_request(
- visit_type, origin_url, privileged_user(request), user_id=request.user.id
+ visit_type,
+ origin_url,
+ privileged_user(request),
+ user_id=request.user.id,
+ **data,
)
-
del sor["id"]
else:
sor = get_save_origin_requests(visit_type, origin_url)
diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py
--- a/swh/web/common/origin_save.py
+++ b/swh/web/common/origin_save.py
@@ -190,7 +190,13 @@
if exists:
size_ = resp.headers.get("Content-Length")
content_length = int(size_) if size_ else None
- last_modified = resp.headers.get("Last-Modified")
+ try:
+ date_str = resp.headers["Last-Modified"]
+ date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z")
+ last_modified = date.isoformat()
+ except (KeyError, ValueError):
+ # if not provided or not parsable, simply keep it None
+ pass
return OriginExistenceCheckInfo(
origin_url=origin_url,
@@ -200,14 +206,18 @@
)
-def _check_origin_exists(origin_url: str) -> None:
+def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo:
"""Ensure the origin exists, if not raise an explicit message."""
- check = origin_exists(origin_url)
- if not check["exists"]:
+ if not origin_url:
+ raise BadInputExc("The origin url provided must be set!")
+ metadata = origin_exists(origin_url)
+ if not metadata["exists"]:
raise BadInputExc(
f"The provided origin url ({escape(origin_url)}) does not exist!"
)
+ return metadata
+
def _get_visit_info_for_save_request(
save_request: SaveOriginRequest,
@@ -354,26 +364,29 @@
origin_url: str,
privileged_user: bool = False,
user_id: Optional[int] = None,
+ **kwargs,
) -> SaveOriginRequestInfo:
"""Create a loading task to save a software origin into the archive.
- This function aims to create a software origin loading task
- trough the use of the swh-scheduler component.
+ This function aims to create a software origin loading task trough the use of the
+ swh-scheduler component.
- First, some checks are performed to see if the visit type and origin
- url are valid but also if the the save request can be accepted.
- If those checks passed, the loading task is then created.
- Otherwise, the save request is put in pending or rejected state.
+ First, some checks are performed to see if the visit type and origin url are valid
+ but also if the the save request can be accepted. For the 'bundle' visit type, this
+ also ensures the artifacts actually exists. If those checks passed, the loading task
+ is then created. Otherwise, the save request is put in pending or rejected state.
- All the submitted save requests are logged into the swh-web
- database to keep track of them.
+ All the submitted save requests are logged into the swh-web database to keep track
+ of them.
Args:
- visit_type: the type of visit to perform (e.g git, hg, svn, ...)
+ visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...)
origin_url: the url of the origin to save
- privileged_user: Whether the user has privileged_user access to extra
- functionality (e.g. bypass save code now review, access to extra visit type)
+ privileged: Whether the user has some more privilege than other (bypass
+ review, access to privileged other visit types)
user_id: User identifier (provided when authenticated)
+ kwargs: Optional parameters (e.g. artifact_url, artifact_filename,
+ artifact_version)
Raises:
BadInputExc: the visit type or origin url is invalid or inexistent
@@ -394,6 +407,11 @@
"""
_check_visit_type_savable(visit_type, privileged_user)
_check_origin_url_valid(origin_url)
+
+ artifact_url = kwargs.get("artifact_url")
+ if visit_type == "bundle":
+ metadata = _check_origin_exists(artifact_url)
+
# if all checks passed so far, we can try and save the origin
save_request_status = can_save_origin(origin_url, privileged_user)
task = None
@@ -402,10 +420,25 @@
# task to load it into the archive
if save_request_status == SAVE_REQUEST_ACCEPTED:
# create a task with high priority
- kwargs = {
+ task_kwargs: Dict[str, Any] = {
"priority": "high",
"url": origin_url,
}
+ if visit_type == "bundle":
+ # extra arguments for that type are required
+ assert metadata is not None
+ task_kwargs = dict(
+ **task_kwargs,
+ artifacts=[
+ {
+ "url": artifact_url,
+ "filename": kwargs["artifact_filename"],
+ "version": kwargs["artifact_version"],
+ "time": metadata["last_modified"],
+ "length": metadata["content_length"],
+ }
+ ],
+ )
sor = None
# get list of previously sumitted save requests
current_sors = list(
@@ -447,7 +480,9 @@
if can_create_task:
# effectively create the scheduler task
- task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs)
+ task_dict = create_oneshot_task_dict(
+ _visit_type_task[visit_type], **task_kwargs
+ )
task = scheduler.create_tasks([task_dict])[0]
# pending save request has been accepted
diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py
--- a/swh/web/common/typing.py
+++ b/swh/web/common/typing.py
@@ -255,4 +255,4 @@
content_length: Optional[int]
"""content length of the artifact"""
last_modified: Optional[str]
- """Last modification time reported by the server"""
+ """Last modification time reported by the server (as iso8601 string)"""
diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html
--- a/swh/web/templates/misc/origin-save.html
+++ b/swh/web/templates/misc/origin-save.html
@@ -1,7 +1,7 @@
{% extends "../layout.html" %}
{% comment %}
-Copyright (C) 2018-2019 The Software Heritage developers
+Copyright (C) 2018-2021 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
@@ -33,7 +33,7 @@
<div class="col-md-1"></div>
<div class="form-group col-md-2">
<label for="swh-input-visit-type">Origin type</label>
- <select id="swh-input-visit-type" class="form-control" required>
+ <select id="swh-input-visit-type" class="form-control" required onchange="swh.save.maybeDisplayExtraInputs();">
{% comment %} <option selected value="">Choose...</option> {% endcomment %}
</select>
<div class="invalid-feedback">The origin type must be specified</div>
@@ -50,6 +50,21 @@
</div>
</div>
</div>
+ <div id="optional-origin-forms" class="form-row"><!-- hidden by span -->
+ <div class="col-md-1"></div>
+ <div class="form-group col-md-6">
+ <label for="swh-input-artifact-url">Artifact url</label>
+ <input type="text" class="form-control" id="swh-input-artifact-url">
+ </div>
+ <div class="form-group col-md-3">
+ <label for="swh-input-artifact-filename">Artifact filename</label>
+ <input type="text" class="form-control" id="swh-input-artifact-filename">
+ </div>
+ <div class="form-group col-md-1">
+ <label for="swh-input-artifact-version">Artifact version</label>
+ <input type="text" class="form-control" id="swh-input-artifact-version">
+ </div>
+ </div>
</form>
<div class="swh-processing-save-request text-center" style="display: none;">
<img src="{% static 'img/swh-spinner.gif' %}">
@@ -123,4 +138,4 @@
swh.save.initOriginSave();
</script>
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py
--- a/swh/web/tests/common/test_origin_save.py
+++ b/swh/web/tests/common/test_origin_save.py
@@ -408,7 +408,20 @@
origin_url=url,
exists=True,
content_length=10,
- last_modified="Sun, 21 Aug 2011 16:26:32 GMT",
+ last_modified="2011-08-21T16:26:32",
+ )
+
+
+def test_origin_exists_200_with_data_unexpected_date_format(requests_mock):
+ """Existing origin should be ok, unexpected last modif time result in no time"""
+ url = "http://example.org/real-url2"
+ requests_mock.head(
+ url, status_code=200, headers={"last-modified": "Sun, 21 Aug 2021 16:26:32",},
+ )
+
+ actual_result = origin_exists(url)
+ assert actual_result == OriginExistenceCheckInfo(
+ origin_url=url, exists=True, content_length=None, last_modified=None,
)

File Metadata

Mime Type
text/plain
Expires
Dec 19 2024, 8:02 PM (11 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3227116

Event Timeline

About · Developer information · Legal