diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py index dfbba133..467aeaba 100644 --- a/swh/web/admin/urls.py +++ b/swh/web/admin/urls.py @@ -1,22 +1,21 @@ # Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.shortcuts import redirect from django.urls import re_path as url from swh.web.admin.adminurls import AdminUrls -import swh.web.admin.deposit # noqa def _admin_default_view(request): return redirect("admin-origin-save-requests") urlpatterns = [ url(r"^$", _admin_default_view, name="admin"), ] urlpatterns += AdminUrls.get_url_patterns() diff --git a/swh/web/common/urlsindex.py b/swh/web/common/urlsindex.py index 1469f54b..21752418 100644 --- a/swh/web/common/urlsindex.py +++ b/swh/web/common/urlsindex.py @@ -1,75 +1,76 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +from collections import defaultdict from typing import Dict, List from django.shortcuts import redirect from django.urls import URLPattern from django.urls import re_path as url -class UrlsIndex(object): +class UrlsIndex: """ Simple helper class for centralizing url patterns of a Django web application. Derived classes should override the 'scope' class attribute otherwise all declared patterns will be grouped under the default one. """ - _urlpatterns: Dict[str, List[URLPattern]] = {} + _urlpatterns: Dict[str, List[URLPattern]] = defaultdict(list) scope = "default" @classmethod def add_url_pattern(cls, url_pattern, view, view_name=None): """ Class method that adds an url pattern to the current scope. Args: url_pattern: regex describing a Django url view: function implementing the Django view view_name: name of the view used to reverse the url """ if cls.scope not in cls._urlpatterns: cls._urlpatterns[cls.scope] = [] if view_name: cls._urlpatterns[cls.scope].append(url(url_pattern, view, name=view_name)) else: cls._urlpatterns[cls.scope].append(url(url_pattern, view)) @classmethod def add_redirect_for_checksum_args(cls, view_name, url_patterns, checksum_args): """ Class method that redirects to view with lowercase checksums when upper/mixed case checksums are passed as url arguments. Args: view_name (str): name of the view to redirect requests url_patterns (List[str]): regexps describing the view urls checksum_args (List[str]): url argument names corresponding to checksum values """ new_view_name = view_name + "-uppercase-checksum" for url_pattern in url_patterns: url_pattern_upper = url_pattern.replace("[0-9a-f]", "[0-9a-fA-F]") def view_redirect(request, *args, **kwargs): for checksum_arg in checksum_args: checksum_upper = kwargs[checksum_arg] kwargs[checksum_arg] = checksum_upper.lower() return redirect(view_name, *args, **kwargs) cls.add_url_pattern(url_pattern_upper, view_redirect, new_view_name) @classmethod def get_url_patterns(cls): """ Class method that returns the list of url pattern associated to the current scope. Returns: The list of url patterns associated to the current scope """ return cls._urlpatterns[cls.scope] diff --git a/swh/web/config.py b/swh/web/config.py index b763dc04..c6b0b135 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,234 +1,235 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict from swh.core import config from swh.counters import get_counters from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SWH_WEB_SERVER_NAME = "archive.softwareheritage.org" SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org" SWH_WEB_STAGING_SERVER_NAMES = [ "webapp.staging.swh.network", "webapp.internal.staging.swh.network", ] SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { "allowed_hosts": ("list", []), "storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10, }, ), "indexer_storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1, }, ), "counters": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5011/", "timeout": 1, }, ), "search": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5010/", "timeout": 10, }, ), "search_config": ( "dict", { "metadata_backend": "swh-indexer-storage", }, # or "swh-search" ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), "serve_assets": ("bool", False), "host": ("string", "127.0.0.1"), "port": ("int", 5004), "secret_key": ("string", "development key"), # do not display code highlighting for content > 1MB "content_display_max_size": ("int", 5 * 1024 * 1024), "snapshot_content_max_size": ("int", 1000), "throttling": ( "dict", { "cache_uri": None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None "scopes": { "swh_api": { "limiter_rate": {"default": "120/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_search": { "limiter_rate": {"default": "10/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_vault_cooking": { "limiter_rate": {"default": "120/h", "GET": "60/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_save_origin": { "limiter_rate": {"default": "120/h", "POST": "10/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_visit_latest": { "limiter_rate": {"default": "700/m"}, "exempted_networks": ["127.0.0.0/8"], }, }, }, ), "vault": ( "dict", { "cls": "remote", "args": { "url": "http://127.0.0.1:5005/", }, }, ), "scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}), "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")), "test_db": ("dict", {"name": "swh-web-test"}), "production_db": ("dict", {"name": "swh-web"}), "deposit": ( "dict", { "private_api_url": "https://deposit.softwareheritage.org/1/private/", "private_api_user": "swhworker", "private_api_password": "some-password", }, ), "e2e_tests_mode": ("bool", False), "es_workers_index_url": ("string", ""), "history_counters_url": ( "string", ( "http://counters1.internal.softwareheritage.org:5011" "/counters_history/history.json" ), ), "client_config": ("dict", {}), "keycloak": ("dict", {"server_url": "", "realm_name": ""}), "graph": ( "dict", { "server_url": "http://graph.internal.softwareheritage.org:5009/graph/", "max_edges": {"staff": 0, "user": 100000, "anonymous": 1000}, }, ), "status": ( "dict", { "server_url": "https://status.softwareheritage.org/", "json_path": "1.0/status/578e5eddcdc0cc7951000520", }, ), "counters_backend": ("string", "swh-storage"), # or "swh-counters" "staging_server_names": ("list", SWH_WEB_STAGING_SERVER_NAMES), "instance_name": ("str", "archive-test.softwareheritage.org"), "give": ("dict", {"public_key": "", "token": ""}), "features": ("dict", {"add_forge_now": True}), "add_forge_now": ("dict", {"email_address": "add-forge-now@example.com"}), "swh_extra_django_apps": ( "list", [ "swh.web.inbound_email", "swh.web.add_forge_now", "swh.web.mailmap", "swh.web.save_code_now", + "swh.web.deposit", ], ), } swhweb_config: Dict[str, Any] = {} def get_config(config_file="web/web"): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get("SWH_CONFIG_FILENAME") if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, "log_dir") if swhweb_config.get("search"): swhweb_config["search"] = get_search(**swhweb_config["search"]) else: swhweb_config["search"] = None swhweb_config["storage"] = get_storage(**swhweb_config["storage"]) swhweb_config["vault"] = get_vault(**swhweb_config["vault"]) swhweb_config["indexer_storage"] = get_indexer_storage( **swhweb_config["indexer_storage"] ) swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) swhweb_config["counters"] = get_counters(**swhweb_config["counters"]) return swhweb_config def search(): """Return the current application's search.""" return get_config()["search"] def storage(): """Return the current application's storage.""" return get_config()["storage"] def vault(): """Return the current application's vault.""" return get_config()["vault"] def indexer_storage(): """Return the current application's indexer storage.""" return get_config()["indexer_storage"] def scheduler(): """Return the current application's scheduler.""" return get_config()["scheduler"] def counters(): """Return the current application's counters.""" return get_config()["counters"] diff --git a/swh/web/deposit/__init__.py b/swh/web/deposit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/templates/admin/deposit.html b/swh/web/deposit/templates/deposit-admin.html similarity index 100% rename from swh/web/templates/admin/deposit.html rename to swh/web/deposit/templates/deposit-admin.html diff --git a/swh/web/admin/deposit.py b/swh/web/deposit/urls.py similarity index 69% rename from swh/web/admin/deposit.py rename to swh/web/deposit/urls.py index 0be07976..84e747d9 100644 --- a/swh/web/admin/deposit.py +++ b/swh/web/deposit/urls.py @@ -1,44 +1,48 @@ # Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import requests from requests.auth import HTTPBasicAuth from django.conf import settings from django.contrib.auth.decorators import user_passes_test from django.http import JsonResponse from django.shortcuts import render +from django.urls import re_path as url -from swh.web.admin.adminurls import admin_route from swh.web.auth.utils import ADMIN_LIST_DEPOSIT_PERMISSION from swh.web.config import get_config -def _can_list_deposits(user): +def can_list_deposits(user): return user.is_staff or user.has_perm(ADMIN_LIST_DEPOSIT_PERMISSION) -@admin_route(r"deposit/", view_name="admin-deposit") -@user_passes_test(_can_list_deposits, login_url=settings.LOGIN_URL) -def _admin_origin_save(request): - return render(request, "admin/deposit.html") +@user_passes_test(can_list_deposits, login_url=settings.LOGIN_URL) +def admin_deposit(request): + return render(request, "deposit-admin.html") -@admin_route(r"deposit/list/", view_name="admin-deposit-list") -@user_passes_test(_can_list_deposits, login_url=settings.LOGIN_URL) -def _admin_deposit_list(request): +@user_passes_test(can_list_deposits, login_url=settings.LOGIN_URL) +def admin_deposit_list(request): config = get_config()["deposit"] private_api_url = config["private_api_url"].rstrip("/") + "/" deposits_list_url = private_api_url + "deposits/datatables/" deposits_list_auth = HTTPBasicAuth( config["private_api_user"], config["private_api_password"] ) deposits = requests.get( deposits_list_url, auth=deposits_list_auth, params=request.GET, timeout=30 ).json() return JsonResponse(deposits) + + +urlpatterns = [ + url(r"^admin/deposit/$", admin_deposit, name="admin-deposit"), + url(r"^admin/deposit/list/$", admin_deposit_list, name="admin-deposit-list"), +] diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index 8ae81521..1f51295b 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,315 +1,315 @@ {% comment %} Copyright (C) 2015-2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block title %}{% endblock %} {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% render_bundle 'guided_tour' %} {{ request.user.is_authenticated|json_script:"swh_user_logged_in" }} {% include "includes/favicon.html" %} {% block header %}{% endblock %} {% if swh_web_prod %} {% endif %}
{% include "misc/hiring-banner.html" %}
{% if swh_web_staging %}
Staging
v{{ swh_web_version }}
{% elif swh_web_dev %}
Development
v{{ swh_web_version|split:"+"|first }}
{% endif %} {% block content %}{% endblock %}
{% include "includes/global-modals.html" %}
back to top
diff --git a/swh/web/tests/deposit/__init__.py b/swh/web/tests/deposit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/tests/deposit/test_app.py b/swh/web/tests/deposit/test_app.py new file mode 100644 index 00000000..27320035 --- /dev/null +++ b/swh/web/tests/deposit/test_app.py @@ -0,0 +1,32 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from django.urls import get_resolver + +from swh.web.common.utils import reverse +from swh.web.deposit.urls import urlpatterns +from swh.web.tests.django_asserts import assert_not_contains +from swh.web.tests.utils import check_html_get_response + + +@pytest.mark.django_db +def test_deposit_deactivate(client, staff_user, django_settings): + """Check Add forge now feature is deactivated when the swh.web.deposit django + application is not in installed apps.""" + + django_settings.SWH_DJANGO_APPS = [ + app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.deposit" + ] + + url = reverse("swh-web-homepage") + client.force_login(staff_user) + resp = check_html_get_response(client, url, status_code=200) + assert_not_contains(resp, "swh-deposit-admin-item") + + add_forge_now_view_names = set(urlpattern.name for urlpattern in urlpatterns) + all_view_names = set(get_resolver().reverse_dict.keys()) + assert add_forge_now_view_names & all_view_names == set() diff --git a/swh/web/tests/admin/test_deposit.py b/swh/web/tests/deposit/test_views.py similarity index 95% rename from swh/web/tests/admin/test_deposit.py rename to swh/web/tests/deposit/test_views.py index 75f64aed..4ae7b12c 100644 --- a/swh/web/tests/admin/test_deposit.py +++ b/swh/web/tests/deposit/test_views.py @@ -1,101 +1,101 @@ # Copyright (C) 2021-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from base64 import b64encode import pytest from swh.web.auth.utils import ADMIN_LIST_DEPOSIT_PERMISSION from swh.web.common.utils import reverse from swh.web.config import get_config from swh.web.tests.utils import ( check_html_get_response, check_http_get_response, create_django_permission, ) def test_deposit_admin_view_not_available_for_anonymous_user(client): url = reverse("admin-deposit") resp = check_html_get_response(client, url, status_code=302) assert resp["location"] == reverse("login", query_params={"next": url}) @pytest.mark.django_db def test_deposit_admin_view_available_for_staff_user(client, staff_user): client.force_login(staff_user) url = reverse("admin-deposit") check_html_get_response( - client, url, status_code=200, template_used="admin/deposit.html" + client, url, status_code=200, template_used="deposit-admin.html" ) @pytest.mark.django_db def test_deposit_admin_view_available_for_user_with_permission(client, regular_user): regular_user.user_permissions.add( create_django_permission(ADMIN_LIST_DEPOSIT_PERMISSION) ) client.force_login(regular_user) url = reverse("admin-deposit") check_html_get_response( - client, url, status_code=200, template_used="admin/deposit.html" + client, url, status_code=200, template_used="deposit-admin.html" ) @pytest.mark.django_db def test_deposit_admin_view_list_deposits(client, staff_user, requests_mock): deposits_data = { "data": [ { "external_id": "hal-02527986", "id": 1066, "raw_metadata": None, "reception_date": "2022-04-08T14:12:34.143000Z", "status": "rejected", "status_detail": None, "swhid": None, "swhid_context": None, "type": "code", "uri": "https://inria.halpreprod.archives-ouvertes.fr/hal-02527986", }, { "external_id": "hal-01243573", "id": 1065, "raw_metadata": None, "reception_date": "2022-04-08T12:53:50.940000Z", "status": "rejected", "status_detail": None, "swhid": None, "swhid_context": None, "type": "code", "uri": "https://inria.halpreprod.archives-ouvertes.fr/hal-01243573", }, ], "draw": 2, "recordsFiltered": 645, "recordsTotal": 1066, } config = get_config()["deposit"] private_api_url = config["private_api_url"].rstrip("/") + "/" deposits_list_url = private_api_url + "deposits/datatables/" basic_auth_payload = ( config["private_api_user"] + ":" + config["private_api_password"] ).encode() requests_mock.get( deposits_list_url, json=deposits_data, request_headers={ "Authorization": f"Basic {b64encode(basic_auth_payload).decode('ascii')}" }, ) client.force_login(staff_user) url = reverse("admin-deposit-list") check_http_get_response( client, url, status_code=200, content_type="application/json" )