diff --git a/assets/src/bundles/save/index.js b/assets/src/bundles/save/index.js
--- a/assets/src/bundles/save/index.js
+++ b/assets/src/bundles/save/index.js
@@ -10,11 +10,23 @@
let saveRequestsTable;
-function originSaveRequest(originType, originUrl,
- acceptedCallback, pendingCallback, errorCallback) {
+function originSaveRequest(
+ originType, originUrl, extraData,
+ acceptedCallback, pendingCallback, errorCallback
+) {
+ // Actually trigger the origin save request
let addSaveOriginRequestUrl = Urls.api_1_save_origin(originType, originUrl);
$('.swh-processing-save-request').css('display', 'block');
- csrfPost(addSaveOriginRequestUrl)
+ let headers = {};
+ let body = null;
+ if (extraData !== {}) {
+ body = JSON.stringify(extraData);
+ headers = {
+ 'Content-Type': 'application/json'
+ };
+ };
+
+ csrfPost(addSaveOriginRequestUrl, headers, body)
.then(handleFetchError)
.then(response => response.json())
.then(data => {
@@ -33,6 +45,14 @@
});
}
+export function maybeDisplayExtraInputs() {
+ // Read the actual selected value and depending on the origin type, display some extra
+ // inputs or hide them.
+ const originType = $('#swh-input-visit-type').val();
+ const display = originType === 'bundle' ? 'flex' : 'none';
+ $('#optional-origin-forms').css('display', display);
+}
+
const userRequestsFilterCheckbox = `
${originType}`);
}
+ // set git as the default value as before
+ $('#swh-input-visit-type').val('git');
});
saveRequestsTable = $('#swh-origin-save-requests')
@@ -220,7 +242,14 @@
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
- originSaveRequest(originType, originUrl,
+ // read the extra inputs for the bundle type
+ let extraData = originType !== 'bundle' ? {} : {
+ 'artifact_url': $('#swh-input-artifact-url').val(),
+ 'artifact_filename': $('#swh-input-artifact-filename').val(),
+ 'artifact_version': $('#swh-input-artifact-version').val()
+ };
+
+ originSaveRequest(originType, originUrl, extraData,
() => $('#swh-origin-save-request-status').html(saveRequestAcceptedAlert),
() => $('#swh-origin-save-request-status').html(saveRequestPendingAlert),
(statusCode, errorData) => {
@@ -347,8 +376,9 @@
let originType = $('#swh-input-visit-type').val();
let originUrl = $('#swh-input-origin-url').val();
+ let extraData = {};
- originSaveRequest(originType, originUrl,
+ originSaveRequest(originType, originUrl, extraData,
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestAcceptedAlert),
() => $('#swh-take-new-snapshot-request-status').html(newSnapshotRequestPendingAlert),
(statusCode, errorData) => {
diff --git a/cypress/integration/origin-save.spec.js b/cypress/integration/origin-save.spec.js
--- a/cypress/integration/origin-save.spec.js
+++ b/cypress/integration/origin-save.spec.js
@@ -19,6 +19,9 @@
'csrfError': 'CSRF Failed: Referrer checking failed - no Referrer.'
};
+const anonymousVisitTypes = ['git', 'hg', 'svn'];
+const allVisitTypes = ['bundle', 'git', 'hg', 'svn'];
+
function makeOriginSaveRequest(originType, originUrl) {
cy.get('#swh-input-origin-url')
.type(originUrl)
@@ -472,4 +475,80 @@
});
+ it('should list unprivileged visit types when not connected', function() {
+ cy.visit(url);
+ cy.get('#swh-input-visit-type').children('option').then(options => {
+ const actual = [...options].map(o => o.value);
+ expect(actual).to.deep.eq(anonymousVisitTypes);
+ });
+ });
+
+ it('should list unprivileged visit types when connected as unprivileged user', function() {
+ cy.userLogin();
+ cy.visit(url);
+ cy.get('#swh-input-visit-type').children('option').then(options => {
+ const actual = [...options].map(o => o.value);
+ expect(actual).to.deep.eq(anonymousVisitTypes);
+ });
+ });
+
+ it('should list privileged visit types when connected as ambassador', function() {
+ cy.ambassadorLogin();
+ cy.visit(url);
+ cy.get('#swh-input-visit-type').children('option').then(options => {
+ const actual = [...options].map(o => o.value);
+ expect(actual).to.deep.eq(allVisitTypes);
+ });
+ });
+
+ it('should display extra inputs when dealing with bundle visit type', function() {
+ cy.ambassadorLogin();
+ cy.visit(url);
+
+ for (let visitType of anonymousVisitTypes) {
+ cy.get('#swh-input-visit-type').select(visitType);
+ cy.get('#optional-origin-forms').should('not.be.visible');
+ }
+
+ // bundle should display more inputs with the bundle type
+ cy.get('#swh-input-visit-type').select('bundle');
+ cy.get('#optional-origin-forms').should('be.visible');
+
+ });
+
+ it('should be allowed to submit bundle save request when connected as ambassador', function() {
+ let originUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf';
+ let artifactUrl = 'https://ftp.gnu.org/pub/pub/gnu/3dldf/3DLDF-1.1.4.tar.gz';
+ let artifactFilename = '3DLDF-1.1.4.tar.gz';
+ let artifactVersion = '1.1.4';
+ stubSaveRequest({
+ requestUrl: this.Urls.api_1_save_origin('bundle', originUrl),
+ saveRequestStatus: 'accepted',
+ originUrl: originUrl,
+ saveTaskStatus: 'not yet scheduled'
+ });
+
+ cy.ambassadorLogin();
+ cy.visit(url);
+
+ // input new bundle information and submit
+ cy.get('#swh-input-origin-url')
+ .type(originUrl)
+ .get('#swh-input-visit-type')
+ .select('bundle')
+ .get('#swh-input-artifact-url')
+ .type(artifactUrl)
+ .get('#swh-input-artifact-filename')
+ .type(artifactFilename)
+ .get('#swh-input-artifact-version')
+ .type(artifactVersion)
+ .get('#swh-save-origin-form')
+ .submit();
+
+ cy.wait('@saveRequest').then(() => {
+ checkAlertVisible('success', saveCodeMsg['success']);
+ });
+
+ });
+
});
diff --git a/cypress/support/index.js b/cypress/support/index.js
--- a/cypress/support/index.js
+++ b/cypress/support/index.js
@@ -53,6 +53,10 @@
return loginUser('user', 'user');
});
+Cypress.Commands.add('ambassadorLogin', () => {
+ return loginUser('ambassador', 'ambassador');
+});
+
before(function() {
this.unarchivedRepo = {
url: 'https://github.com/SoftwareHeritage/swh-web',
diff --git a/swh/web/api/views/origin_save.py b/swh/web/api/views/origin_save.py
--- a/swh/web/api/views/origin_save.py
+++ b/swh/web/api/views/origin_save.py
@@ -5,7 +5,7 @@
from swh.web.api.apidoc import api_doc, format_docstring
from swh.web.api.apiurls import api_route
-from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION
+from swh.web.auth.utils import privileged_user
from swh.web.common.origin_save import (
create_save_origin_request,
get_save_origin_requests,
@@ -83,12 +83,14 @@
"""
+ data = request.data or {}
if request.method == "POST":
- bypass_pending_review = request.user.is_authenticated and request.user.has_perm(
- SWH_AMBASSADOR_PERMISSION
- )
sor = create_save_origin_request(
- visit_type, origin_url, bypass_pending_review, user_id=request.user.id
+ visit_type,
+ origin_url,
+ privileged_user(request),
+ user_id=request.user.id,
+ **data,
)
del sor["id"]
else:
diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py
--- a/swh/web/auth/utils.py
+++ b/swh/web/auth/utils.py
@@ -70,3 +70,14 @@
The decrypted data
"""
return _get_fernet(password, salt).decrypt(data)
+
+
+def privileged_user(request) -> bool:
+ """Determine whether a user is authenticated and is a privileged one (e.g ambassador).
+ This allows such user to have access to some more actions (e.g. bypass save code now
+ review, access to 'bundle' type...)
+
+ """
+ return request.user.is_authenticated and request.user.has_perm(
+ SWH_AMBASSADOR_PERMISSION
+ )
diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py
--- a/swh/web/common/origin_save.py
+++ b/swh/web/common/origin_save.py
@@ -113,6 +113,10 @@
# TODO: do not hardcode the task name here (T1157)
_visit_type_task = {"git": "load-git", "hg": "load-hg", "svn": "load-svn"}
+_visit_type_task_privileged = {
+ "bundle": "load-archive-files",
+}
+
# map scheduler task status to origin save status
_save_task_status = {
@@ -134,23 +138,47 @@
}
-def get_savable_visit_types() -> List[str]:
+def get_savable_visit_types_dict(privileged_user: bool = False) -> Dict:
+ """Returned the supported task types the user has access to.
+
+ Args:
+ privileged_user: Flag to determine if all visit types should be returned or not.
+ Default to False to only list unprivileged visit types.
+
+ Returns:
+ the dict of supported visit types for the user
+
"""
- Get the list of visit types that can be performed
- through a save request.
+ if privileged_user:
+ task_types = {**_visit_type_task, **_visit_type_task_privileged}
+ else:
+ task_types = _visit_type_task
+
+ return task_types
+
+
+def get_savable_visit_types(privileged_user: bool = False) -> List[str]:
+ """Return the list of visit types the user can perform save requests on.
+
+ Args:
+ privileged_user: Flag to determine if all visit types should be returned or not.
+ Default to False to only list unprivileged visit types.
Returns:
- list: the list of saveable visit types
+ the list of saveable visit types
+
"""
- return sorted(list(_visit_type_task.keys()))
+ return sorted(list(get_savable_visit_types_dict(privileged_user).keys()))
-def _check_visit_type_savable(visit_type: str) -> None:
- allowed_visit_types = ", ".join(get_savable_visit_types())
- if visit_type not in _visit_type_task:
+
+def _check_visit_type_savable(visit_type: str, privileged_user: bool = False) -> None:
+ visit_type_tasks = get_savable_visit_types(privileged_user)
+ if visit_type not in visit_type_tasks:
+ allowed_visit_types = ", ".join(visit_type_tasks)
raise BadInputExc(
- "Visit of type %s can not be saved! "
- "Allowed types are the following: %s" % (visit_type, allowed_visit_types)
+ f"Visit of type {visit_type} can not be saved! "
+ f"Allowed types are the following: {allowed_visit_types}"
)
@@ -178,7 +206,13 @@
if exists:
size_ = resp.headers.get("Content-Length")
content_length = int(size_) if size_ else None
- last_modified = resp.headers.get("Last-Modified")
+ try:
+ date_str = resp.headers["Last-Modified"]
+ date = datetime.strptime(date_str, "%a, %d %b %Y %H:%M:%S %Z")
+ last_modified = date.isoformat()
+ except (KeyError, ValueError):
+ # if not provided or not parsable as per the expected format, keep it None
+ pass
return OriginExistenceCheckInfo(
origin_url=origin_url,
@@ -188,14 +222,18 @@
)
-def _check_origin_exists(origin_url: str) -> None:
+def _check_origin_exists(origin_url: Optional[str]) -> OriginExistenceCheckInfo:
"""Ensure the origin exists, if not raise an explicit message."""
- check = origin_exists(origin_url)
- if not check["exists"]:
+ if not origin_url:
+ raise BadInputExc("The origin url provided must be set!")
+ metadata = origin_exists(origin_url)
+ if not metadata["exists"]:
raise BadInputExc(
f"The provided origin url ({escape(origin_url)}) does not exist!"
)
+ return metadata
+
def _get_visit_info_for_save_request(
save_request: SaveOriginRequest,
@@ -340,26 +378,31 @@
def create_save_origin_request(
visit_type: str,
origin_url: str,
- bypass_pending_review: bool = False,
+ privileged_user: bool = False,
user_id: Optional[int] = None,
+ **kwargs,
) -> SaveOriginRequestInfo:
- """
- Create a loading task to save a software origin into the archive.
+ """Create a loading task to save a software origin into the archive.
- This function aims to create a software origin loading task
- trough the use of the swh-scheduler component.
+ This function aims to create a software origin loading task trough the use of the
+ swh-scheduler component.
- First, some checks are performed to see if the visit type and origin
- url are valid but also if the the save request can be accepted.
- If those checks passed, the loading task is then created.
- Otherwise, the save request is put in pending or rejected state.
+ First, some checks are performed to see if the visit type and origin url are valid
+ but also if the the save request can be accepted. For the 'bundle' visit type, this
+ also ensures the artifacts actually exists. If those checks passed, the loading task
+ is then created. Otherwise, the save request is put in pending or rejected state.
- All the submitted save requests are logged into the swh-web
- database to keep track of them.
+ All the submitted save requests are logged into the swh-web database to keep track
+ of them.
Args:
- visit_type: the type of visit to perform (e.g git, hg, svn, ...)
+ visit_type: the type of visit to perform (e.g. git, hg, svn, bundle, ...)
origin_url: the url of the origin to save
+ privileged: Whether the user has some more privilege than other (bypass
+ review, access to privileged other visit types)
+ user_id: User identifier (provided when authenticated)
+ kwargs: Optional parameters (e.g. artifact_url, artifact_filename,
+ artifact_version)
Raises:
BadInputExc: the visit type or origin url is invalid or inexistent
@@ -377,22 +420,42 @@
**not created**, **not yet scheduled**, **scheduled**,
**succeed** or **failed**
-
"""
- _check_visit_type_savable(visit_type)
+ visit_type_tasks = get_savable_visit_types_dict(privileged_user)
+ _check_visit_type_savable(visit_type, privileged_user)
_check_origin_url_valid(origin_url)
+
+ artifact_url = kwargs.get("artifact_url")
+ if visit_type == "bundle":
+ metadata = _check_origin_exists(artifact_url)
+
# if all checks passed so far, we can try and save the origin
- save_request_status = can_save_origin(origin_url, bypass_pending_review)
+ save_request_status = can_save_origin(origin_url, privileged_user)
task = None
# if the origin save request is accepted, create a scheduler
# task to load it into the archive
if save_request_status == SAVE_REQUEST_ACCEPTED:
# create a task with high priority
- kwargs = {
+ task_kwargs: Dict[str, Any] = {
"priority": "high",
"url": origin_url,
}
+ if visit_type == "bundle":
+ # extra arguments for that type are required
+ assert metadata is not None
+ task_kwargs = dict(
+ **task_kwargs,
+ artifacts=[
+ {
+ "url": artifact_url,
+ "filename": kwargs["artifact_filename"],
+ "version": kwargs["artifact_version"],
+ "time": metadata["last_modified"],
+ "length": metadata["content_length"],
+ }
+ ],
+ )
sor = None
# get list of previously sumitted save requests
current_sors = list(
@@ -434,7 +497,10 @@
if can_create_task:
# effectively create the scheduler task
- task_dict = create_oneshot_task_dict(_visit_type_task[visit_type], **kwargs)
+ task_dict = create_oneshot_task_dict(
+ visit_type_tasks[visit_type], **task_kwargs
+ )
+
task = scheduler.create_tasks([task_dict])[0]
# pending save request has been accepted
@@ -450,6 +516,7 @@
loading_task_id=task["id"],
user_ids=f'"{user_id}"' if user_id else None,
)
+
# save request must be manually reviewed for acceptation
elif save_request_status == SAVE_REQUEST_PENDING:
# check if there is already such a save request already submitted,
@@ -778,7 +845,8 @@
SAVE_TASK_RUNNING,
)
- visit_types = get_savable_visit_types()
+ # for metrics, we want access to all visit types
+ visit_types = get_savable_visit_types(privileged_user=True)
labels_set = product(request_statuses, visit_types)
diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py
--- a/swh/web/common/swh_templatetags.py
+++ b/swh/web/common/swh_templatetags.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2019 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -125,15 +125,16 @@
@register.filter
-def visit_type_savable(visit_type):
+def visit_type_savable(visit_type: str) -> bool:
"""Django template filter to check if a save request can be
created for a given visit type.
Args:
- visit_type (str): the type of visit
+ visit_type: the type of visit
Returns:
If the visit type is saveable or not
+
"""
return visit_type in get_savable_visit_types()
diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py
--- a/swh/web/common/typing.py
+++ b/swh/web/common/typing.py
@@ -255,4 +255,4 @@
content_length: Optional[int]
"""content length of the artifact"""
last_modified: Optional[str]
- """Last modification time reported by the server"""
+ """Last modification time reported by the server (as iso8601 string)"""
diff --git a/swh/web/misc/origin_save.py b/swh/web/misc/origin_save.py
--- a/swh/web/misc/origin_save.py
+++ b/swh/web/misc/origin_save.py
@@ -9,6 +9,7 @@
from django.http import JsonResponse
from django.shortcuts import render
+from swh.web.auth.utils import privileged_user
from swh.web.common.models import SaveOriginRequest
from swh.web.common.origin_save import (
get_savable_visit_types,
@@ -24,8 +25,11 @@
)
-def _visit_save_types_list(request):
- visit_types = get_savable_visit_types()
+def _visit_save_types_list(request) -> JsonResponse:
+ """Return the list of supported visit types as json response
+
+ """
+ visit_types = get_savable_visit_types(privileged_user(request))
return JsonResponse(visit_types, safe=False)
diff --git a/swh/web/templates/misc/origin-save.html b/swh/web/templates/misc/origin-save.html
--- a/swh/web/templates/misc/origin-save.html
+++ b/swh/web/templates/misc/origin-save.html
@@ -1,7 +1,7 @@
{% extends "../layout.html" %}
{% comment %}
-Copyright (C) 2018-2019 The Software Heritage developers
+Copyright (C) 2018-2021 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
@@ -33,7 +33,7 @@
-
+
+
+
Artifact information:
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -126,4 +142,4 @@
swh.save.initOriginSave();
-{% endblock %}
\ No newline at end of file
+{% endblock %}
diff --git a/swh/web/tests/api/test_throttling.py b/swh/web/tests/api/test_throttling.py
--- a/swh/web/tests/api/test_throttling.py
+++ b/swh/web/tests/api/test_throttling.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2017-2020 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -6,8 +6,7 @@
import pytest
from django.conf.urls import url
-from django.contrib.auth.models import Permission, User
-from django.contrib.contenttypes.models import ContentType
+from django.contrib.auth.models import User
from django.test.utils import override_settings
from rest_framework.decorators import api_view
from rest_framework.response import Response
@@ -27,6 +26,7 @@
scope3_limiter_rate,
scope3_limiter_rate_post,
)
+from swh.web.tests.utils import create_django_permission
from swh.web.urls import urlpatterns
@@ -217,14 +217,7 @@
@pytest.mark.django_db
def test_users_with_throttling_exempted_perm_are_not_rate_limited(api_client):
user = User.objects.create_user(username="johndoe", password="")
- perm_splitted = API_THROTTLING_EXEMPTED_PERM.split(".")
- app_label = ".".join(perm_splitted[:-1])
- perm_name = perm_splitted[-1]
- content_type = ContentType.objects.create(app_label=app_label, model="dummy")
- permission = Permission.objects.create(
- codename=perm_name, name=perm_name, content_type=content_type,
- )
- user.user_permissions.add(permission)
+ user.user_permissions.add(create_django_permission(API_THROTTLING_EXEMPTED_PERM))
assert user.has_perm(API_THROTTLING_EXEMPTED_PERM)
diff --git a/swh/web/tests/api/views/test_origin_save.py b/swh/web/tests/api/views/test_origin_save.py
--- a/swh/web/tests/api/views/test_origin_save.py
+++ b/swh/web/tests/api/views/test_origin_save.py
@@ -460,7 +460,77 @@
SaveAuthorizedOrigin.objects.get(url=origin_to_review)
-def test_create_save_request_accepted_ambassador_user(
+def test_create_save_request_bundle_with_ambassador_user(
+ api_client, origin_to_review, keycloak_oidc, mocker, requests_mock,
+):
+
+ keycloak_oidc.realm_permissions = [SWH_AMBASSADOR_PERMISSION]
+ oidc_profile = keycloak_oidc.login()
+ api_client.credentials(HTTP_AUTHORIZATION=f"Bearer {oidc_profile['refresh_token']}")
+
+ originUrl = "https://somewhere.org/simple"
+ artifact_version = "1.2.3"
+ artifact_filename = f"tarball-{artifact_version}.tar.gz"
+ artifact_url = f"{originUrl}/{artifact_filename}"
+ content_length = "100"
+ last_modified = "Sun, 21 Aug 2011 16:26:32 GMT"
+
+ requests_mock.head(
+ artifact_url,
+ status_code=200,
+ headers={"content-length": content_length, "last-modified": last_modified,},
+ )
+
+ mock_scheduler = mocker.patch("swh.web.common.origin_save.scheduler")
+ mock_scheduler.get_task_runs.return_value = []
+ mock_scheduler.create_tasks.return_value = [
+ {
+ "id": 10,
+ "priority": "high",
+ "policy": "oneshot",
+ "status": "next_run_not_scheduled",
+ "type": "load-archive-files",
+ "arguments": {
+ "args": [],
+ "kwargs": {
+ "url": originUrl,
+ "artifacts": [
+ {
+ "url": artifact_url,
+ "filename": artifact_filename,
+ "version": artifact_version,
+ "time": last_modified,
+ "length": content_length,
+ }
+ ],
+ },
+ },
+ },
+ ]
+
+ # then
+ url = reverse(
+ "api-1-save-origin",
+ url_args={"visit_type": "bundle", "origin_url": originUrl,},
+ )
+
+ response = check_api_post_response(
+ api_client,
+ url,
+ status_code=200,
+ data={
+ "artifact_url": artifact_url,
+ "artifact_filename": artifact_filename,
+ "artifact_version": artifact_version,
+ },
+ )
+
+ assert response.data["save_request_status"] == SAVE_REQUEST_ACCEPTED
+
+ assert SaveAuthorizedOrigin.objects.get(url=originUrl)
+
+
+def test_create_save_request_bundle_accepted_ambassador_user(
api_client, origin_to_review, keycloak_oidc, mocker
):
diff --git a/swh/web/tests/common/test_origin_save.py b/swh/web/tests/common/test_origin_save.py
--- a/swh/web/tests/common/test_origin_save.py
+++ b/swh/web/tests/common/test_origin_save.py
@@ -24,6 +24,10 @@
)
from swh.web.common.origin_save import (
_check_origin_exists,
+ _check_visit_type_savable,
+ _visit_type_task,
+ _visit_type_task_privileged,
+ get_savable_visit_types,
get_save_origin_requests,
get_save_origin_task_info,
origin_exists,
@@ -105,6 +109,34 @@
return task, task_run
+@pytest.mark.parametrize(
+ "wrong_type,privileged_user",
+ [
+ ("dummy", True),
+ ("dumb", False),
+ ("bundle", False), # when no privilege, this is rejected
+ ],
+)
+def test__check_visit_type_savable(wrong_type, privileged_user):
+
+ with pytest.raises(BadInputExc, match="Allowed types"):
+ _check_visit_type_savable(wrong_type, privileged_user)
+
+ # when privileged_user, the following is accepted though
+ _check_visit_type_savable("bundle", True)
+
+
+def test_get_savable_visit_types():
+ default_list = list(_visit_type_task.keys())
+
+ assert set(get_savable_visit_types()) == set(default_list)
+
+ privileged_list = default_list.copy()
+ privileged_list += list(_visit_type_task_privileged.keys())
+
+ assert set(get_savable_visit_types(privileged_user=True)) == set(privileged_list)
+
+
def _get_save_origin_task_info_test(
mocker, task_archived=False, es_available=True, full_info=True
):
@@ -327,12 +359,21 @@
_check_origin_exists(url_ko)
+@pytest.mark.parametrize("invalid_origin", [None, ""])
+def test__check_origin_invalid_input(invalid_origin):
+ with pytest.raises(BadInputExc, match="must be set"):
+ _check_origin_exists(invalid_origin)
+
+
def test__check_origin_exists_200(requests_mock):
url = "https://example.org/url"
requests_mock.head(url, status_code=200)
# passes the check
- _check_origin_exists(url)
+ actual_metadata = _check_origin_exists(url)
+
+ # and we actually may have retrieved some metadata on the origin
+ assert actual_metadata == origin_exists(url)
def test_origin_exists_404(requests_mock):
@@ -376,7 +417,23 @@
origin_url=url,
exists=True,
content_length=10,
- last_modified="Sun, 21 Aug 2011 16:26:32 GMT",
+ last_modified="2011-08-21T16:26:32",
+ )
+
+
+def test_origin_exists_200_with_data_unexpected_date_format(requests_mock):
+ """Existing origin should be ok, unexpected last modif time result in no time"""
+ url = "http://example.org/real-url2"
+ # this is parsable but not as expected
+ unexpected_format_date = "Sun, 21 Aug 2021 16:26:32"
+ requests_mock.head(
+ url, status_code=200, headers={"last-modified": unexpected_format_date,},
+ )
+
+ actual_result = origin_exists(url)
+ # so the resulting date is None
+ assert actual_result == OriginExistenceCheckInfo(
+ origin_url=url, exists=True, content_length=None, last_modified=None,
)
diff --git a/swh/web/tests/create_test_users.py b/swh/web/tests/create_test_users.py
--- a/swh/web/tests/create_test_users.py
+++ b/swh/web/tests/create_test_users.py
@@ -3,14 +3,27 @@
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
+from typing import Dict, List, Tuple
from django.contrib.auth import get_user_model
+from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION
+from swh.web.tests.utils import create_django_permission
+
User = get_user_model()
-username = "user"
-password = "user"
-email = "user@swh-web.org"
-if not User.objects.filter(username=username).exists():
- User.objects.create_user(username, email, password)
+users: Dict[str, Tuple[str, str, List[str]]] = {
+ "user": ("user", "user@swh-web.org", []),
+ "ambassador": ("ambassador", "ambassador@swh-web.org", [SWH_AMBASSADOR_PERMISSION]),
+}
+
+for username, (password, email, permissions) in users.items():
+ if not User.objects.filter(username=username).exists():
+ user = User.objects.create_user(username, email, password)
+ if permissions:
+ for perm_name in permissions:
+ permission = create_django_permission(perm_name)
+ user.user_permissions.add(permission)
+
+ user.save()
diff --git a/swh/web/tests/misc/test_origin_save.py b/swh/web/tests/misc/test_origin_save.py
--- a/swh/web/tests/misc/test_origin_save.py
+++ b/swh/web/tests/misc/test_origin_save.py
@@ -9,11 +9,15 @@
import pytest
from swh.auth.django.utils import oidc_user_from_profile
+from swh.web.auth.utils import SWH_AMBASSADOR_PERMISSION
from swh.web.common.models import SaveOriginRequest
from swh.web.common.origin_save import SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEEDED
from swh.web.common.utils import reverse
from swh.web.tests.utils import check_http_get_response
+VISIT_TYPES = ("git", "svn", "hg")
+PRIVILEGED_VISIT_TYPES = tuple(list(VISIT_TYPES) + ["bundle"])
+
def test_old_save_url_redirection(client):
url = reverse("browse-origin-save")
@@ -23,11 +27,36 @@
assert resp["location"] == redirect_url
+def test_save_types_list_default(client):
+ """Unprivileged listing should display default list of visit types.
+
+ """
+ url = reverse("origin-save-types-list")
+ resp = check_http_get_response(client, url, status_code=200)
+
+ actual_response = resp.json()
+ assert set(actual_response) == set(VISIT_TYPES)
+
+
+@pytest.mark.django_db
+def test_save_types_list_privileged(client, keycloak_oidc):
+ """Privileged listing should display all visit types.
+
+ """
+ keycloak_oidc.realm_permissions = [SWH_AMBASSADOR_PERMISSION]
+ client.login(code="", code_verifier="", redirect_uri="")
+
+ url = reverse("origin-save-types-list")
+ resp = check_http_get_response(client, url, status_code=200)
+
+ actual_response = resp.json()
+ assert set(actual_response) == set(PRIVILEGED_VISIT_TYPES)
+
+
@pytest.mark.django_db
-def test_save_origin_requests_list(client, mocker, keycloak_oidc):
- visit_types = ("git", "svn", "hg")
+def test_save_origin_requests_list(client, mocker):
nb_origins_per_type = 10
- for visit_type in visit_types:
+ for visit_type in VISIT_TYPES:
for i in range(nb_origins_per_type):
SaveOriginRequest.objects.create(
request_date=datetime.now(tz=timezone.utc),
@@ -45,7 +74,7 @@
# retrieve all save requests in 3 pages, sorted in descending order
# of request creation
- for i, visit_type in enumerate(reversed(visit_types)):
+ for i, visit_type in enumerate(reversed(VISIT_TYPES)):
url = reverse(
"origin-save-requests-list",
url_args={"status": "all"},
@@ -65,13 +94,13 @@
)
sors = json.loads(resp.content.decode("utf-8"))
assert sors["draw"] == i + 1
- assert sors["recordsFiltered"] == len(visit_types) * nb_origins_per_type
- assert sors["recordsTotal"] == len(visit_types) * nb_origins_per_type
+ assert sors["recordsFiltered"] == len(VISIT_TYPES) * nb_origins_per_type
+ assert sors["recordsTotal"] == len(VISIT_TYPES) * nb_origins_per_type
assert len(sors["data"]) == nb_origins_per_type
assert all(d["visit_type"] == visit_type for d in sors["data"])
# retrieve save requests filtered by visit type in a single page
- for i, visit_type in enumerate(reversed(visit_types)):
+ for i, visit_type in enumerate(reversed(VISIT_TYPES)):
url = reverse(
"origin-save-requests-list",
url_args={"status": "all"},
@@ -92,7 +121,7 @@
sors = json.loads(resp.content.decode("utf-8"))
assert sors["draw"] == i + 1
assert sors["recordsFiltered"] == nb_origins_per_type
- assert sors["recordsTotal"] == len(visit_types) * nb_origins_per_type
+ assert sors["recordsTotal"] == len(VISIT_TYPES) * nb_origins_per_type
assert len(sors["data"]) == nb_origins_per_type
assert all(d["visit_type"] == visit_type for d in sors["data"])
diff --git a/swh/web/tests/test_create_users.py b/swh/web/tests/test_create_users.py
new file mode 100644
--- /dev/null
+++ b/swh/web/tests/test_create_users.py
@@ -0,0 +1,16 @@
+# Copyright (C) 2021 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+
+def test_create_users_test_users_exist(db):
+ from .create_test_users import User, users
+
+ for username, (_, _, permissions) in users.items():
+
+ user = User.objects.filter(username=username).get()
+ assert user is not None
+
+ for permission in permissions:
+ assert user.has_perm(permission)
diff --git a/swh/web/tests/utils.py b/swh/web/tests/utils.py
--- a/swh/web/tests/utils.py
+++ b/swh/web/tests/utils.py
@@ -1,10 +1,12 @@
-# Copyright (C) 2020 The Software Heritage developers
+# Copyright (C) 2020-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from typing import Any, Dict, Optional, cast
+from django.contrib.auth.models import Permission
+from django.contrib.contenttypes.models import ContentType
from django.http import HttpResponse, StreamingHttpResponse
from django.test.client import Client
from rest_framework.response import Response
@@ -207,3 +209,23 @@
if template_used is not None:
assert_template_used(response, template_used)
return response
+
+
+def create_django_permission(perm_name: str) -> Permission:
+ """Create permission out of a permission name string
+
+ Args:
+ perm_name: Permission name (e.g. swh.web.api.throttling_exempted,
+ swh.ambassador, ...)
+
+ Returns:
+ The persisted permission
+
+ """
+ perm_splitted = perm_name.split(".")
+ app_label = ".".join(perm_splitted[:-1])
+ perm_name = perm_splitted[-1]
+ content_type = ContentType.objects.create(app_label=app_label, model="dummy")
+ return Permission.objects.create(
+ codename=perm_name, name=perm_name, content_type=content_type,
+ )