diff --git a/swh/web/admin/add_forge_now.py b/swh/web/add_forge_now/admin_views.py similarity index 77% rename from swh/web/admin/add_forge_now.py rename to swh/web/add_forge_now/admin_views.py index 64a1e260..bdaefbc7 100644 --- a/swh/web/admin/add_forge_now.py +++ b/swh/web/add_forge_now/admin_views.py @@ -1,44 +1,35 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf import settings from django.contrib.auth.decorators import user_passes_test from django.shortcuts import render from swh.web.add_forge_now.models import RequestStatus -from swh.web.admin.adminurls import admin_route from swh.web.auth.utils import is_add_forge_now_moderator -@admin_route( - r"add-forge/requests/", - view_name="add-forge-now-requests-moderation", -) @user_passes_test(is_add_forge_now_moderator, login_url=settings.LOGIN_URL) def add_forge_now_requests_moderation_dashboard(request): """Moderation dashboard to allow listing current requests.""" return render( request, - "add_forge_now/requests-moderation.html", + "add-forge-requests-moderation.html", {"heading": "Add forge now requests moderation"}, ) -@admin_route( - r"add-forge/request/(?P(\d)+)/", - view_name="add-forge-now-request-dashboard", -) @user_passes_test(is_add_forge_now_moderator, login_url=settings.LOGIN_URL) def add_forge_now_request_dashboard(request, request_id): """Moderation dashboard to allow listing current requests.""" return render( request, - "add_forge_now/request-dashboard.html", + "add-forge-request-dashboard.html", { "request_id": request_id, "heading": "Add forge now request dashboard", "next_statuses_for": RequestStatus.next_statuses_str(), }, ) diff --git a/swh/web/api/views/add_forge_now.py b/swh/web/add_forge_now/api_views.py similarity index 100% rename from swh/web/api/views/add_forge_now.py rename to swh/web/add_forge_now/api_views.py diff --git a/swh/web/templates/add_forge_now/common.html b/swh/web/add_forge_now/templates/add-forge-common.html similarity index 98% rename from swh/web/templates/add_forge_now/common.html rename to swh/web/add_forge_now/templates/add-forge-common.html index 24d34466..fed2359d 100644 --- a/swh/web/templates/add_forge_now/common.html +++ b/swh/web/add_forge_now/templates/add-forge-common.html @@ -1,72 +1,72 @@ -{% extends "../layout.html" %} +{% extends "layout.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% load static %} {% block header %} {% render_bundle 'add_forge' %} {% endblock %} {% block title %} Add forge now – Software Heritage archive {% endblock %} {% block navbar-content %}

Request the addition of a forge into the archive

{% endblock %} {% block content %}

“Add forge now” provides a service for Software Heritage users to save a complete forge in the Software Heritage archive by requesting the addition of the forge URL into the list of regularly visited forges.

{% if not user.is_authenticated %}

You can submit an “Add forge now” request only when you are authenticated, please login to submit the request.

{% endif %}
{% block tab_content %} {% endblock %}
{% endblock %} diff --git a/swh/web/templates/add_forge_now/creation_form.html b/swh/web/add_forge_now/templates/add-forge-creation-form.html similarity index 99% rename from swh/web/templates/add_forge_now/creation_form.html rename to swh/web/add_forge_now/templates/add-forge-creation-form.html index 6be8f222..86e65b2c 100644 --- a/swh/web/templates/add_forge_now/creation_form.html +++ b/swh/web/add_forge_now/templates/add-forge-creation-form.html @@ -1,129 +1,129 @@ -{% extends "./common.html" %} +{% extends "./add-forge-common.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% block tab_content %}
{% if not user.is_authenticated %}

You must be logged in to submit an add forge request. Please log in

{% else %}
{% csrf_token %}
Supported forge types in software archive.
Remote URL of the forge.
Name of the forge administrator.
Email of the forge administrator. The given email address will not be used for any purpose outside the “add forge now” process.
Optionally, leave a comment to the moderator regarding your request.

Once an add-forge-request is submitted, its status can be viewed in the submitted requests list. This process involves a moderator approval and might take a few days to handle (it primarily depends on the response time from the forge).

{% endif %}
{% endblock %} diff --git a/swh/web/templates/add_forge_now/help.html b/swh/web/add_forge_now/templates/add-forge-help.html similarity index 98% rename from swh/web/templates/add_forge_now/help.html rename to swh/web/add_forge_now/templates/add-forge-help.html index 9b4d7092..4f89a646 100644 --- a/swh/web/templates/add_forge_now/help.html +++ b/swh/web/add_forge_now/templates/add-forge-help.html @@ -1,89 +1,89 @@ -{% extends "./common.html" %} +{% extends "./add-forge-common.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% block tab_content %}

For submitting an "Add forge now" request", you have to provide the following details:

Once submitted, your "add forge" request can be in one of the following states

{% endblock %} diff --git a/swh/web/templates/add_forge_now/list.html b/swh/web/add_forge_now/templates/add-forge-list.html similarity index 94% rename from swh/web/templates/add_forge_now/list.html rename to swh/web/add_forge_now/templates/add-forge-list.html index eb33fa69..1987168f 100644 --- a/swh/web/templates/add_forge_now/list.html +++ b/swh/web/add_forge_now/templates/add-forge-list.html @@ -1,24 +1,24 @@ -{% extends "./common.html" %} +{% extends "./add-forge-common.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% block tab_content %}
Submission date Forge type Forge URL Status
{% endblock %} diff --git a/swh/web/templates/add_forge_now/request-dashboard.html b/swh/web/add_forge_now/templates/add-forge-request-dashboard.html similarity index 99% rename from swh/web/templates/add_forge_now/request-dashboard.html rename to swh/web/add_forge_now/templates/add-forge-request-dashboard.html index c263b910..61987e38 100644 --- a/swh/web/templates/add_forge_now/request-dashboard.html +++ b/swh/web/add_forge_now/templates/add-forge-request-dashboard.html @@ -1,122 +1,122 @@ -{% extends "../layout.html" %} +{% extends "layout.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% load static %} {% load swh_templatetags %} {% block header %} {% render_bundle 'add_forge' %} {% endblock %} {% block title %}{{heading}} – Software Heritage archive{% endblock %} {% block navbar-content %}

Add forge now request dashboard

{% endblock %} {% block content %}

Error fetching information about the request

{% csrf_token %}
Enter a comment related to your decision.

Request status
Forge type
Forge URL
Contact name
Consent to use name
Contact email
Message

{% endblock %} diff --git a/swh/web/templates/add_forge_now/requests-moderation.html b/swh/web/add_forge_now/templates/add-forge-requests-moderation.html similarity index 97% rename from swh/web/templates/add_forge_now/requests-moderation.html rename to swh/web/add_forge_now/templates/add-forge-requests-moderation.html index 6d3010d1..dcdbe7b1 100644 --- a/swh/web/templates/add_forge_now/requests-moderation.html +++ b/swh/web/add_forge_now/templates/add-forge-requests-moderation.html @@ -1,48 +1,48 @@ -{% extends "../layout.html" %} +{% extends "layout.html" %} {% comment %} Copyright (C) 2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load render_bundle from webpack_loader %} {% load static %} {% block header %} {% render_bundle 'add_forge' %} {% endblock %} {% block title %}{{ heading }} – Software Heritage archive{% endblock %} {% block navbar-content %}

Add forge now moderation

{% endblock %} {% block content %}
ID Submission date Forge type Forge URL Moderator Name Last Modified Date Status

{% endblock %} diff --git a/swh/web/add_forge_now/urls.py b/swh/web/add_forge_now/urls.py new file mode 100644 index 00000000..e8b1423e --- /dev/null +++ b/swh/web/add_forge_now/urls.py @@ -0,0 +1,47 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.urls import re_path as url + +from swh.web.add_forge_now.admin_views import ( + add_forge_now_request_dashboard, + add_forge_now_requests_moderation_dashboard, +) + +# register Web API endpoints +import swh.web.add_forge_now.api_views # noqa +from swh.web.add_forge_now.views import ( + add_forge_request_list_datatables, + create_request_create, + create_request_help, + create_request_list, + create_request_message_source, +) + +urlpatterns = [ + url( + r"^add-forge/request/list/datatables/$", + add_forge_request_list_datatables, + name="add-forge-request-list-datatables", + ), + url(r"^add-forge/request/create/$", create_request_create, name="forge-add-create"), + url(r"^add-forge/request/list/$", create_request_list, name="forge-add-list"), + url( + r"^add-forge/request/message-source/(?P\d+)/$", + create_request_message_source, + name="forge-add-message-source", + ), + url(r"^add-forge/request/help/$", create_request_help, name="forge-add-help"), + url( + r"^admin/add-forge/requests/$", + add_forge_now_requests_moderation_dashboard, + name="add-forge-now-requests-moderation", + ), + url( + r"^admin/add-forge/request/(?P(\d)+)/$", + add_forge_now_request_dashboard, + name="add-forge-now-request-dashboard", + ), +] diff --git a/swh/web/add_forge_now/views.py b/swh/web/add_forge_now/views.py index c17fdb06..a2f41c10 100644 --- a/swh/web/add_forge_now/views.py +++ b/swh/web/add_forge_now/views.py @@ -1,160 +1,142 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Any, Dict, List from django.conf import settings from django.contrib.auth.decorators import user_passes_test from django.core.paginator import Paginator from django.db.models import Q from django.http.request import HttpRequest from django.http.response import HttpResponse, JsonResponse from django.shortcuts import render -from django.urls import re_path as url -from swh.web.add_forge_now.models import Request as AddForgeRequest -from swh.web.add_forge_now.models import RequestHistory -from swh.web.api.views.add_forge_now import ( +from swh.web.add_forge_now.api_views import ( AddForgeNowRequestPublicSerializer, AddForgeNowRequestSerializer, ) +from swh.web.add_forge_now.models import Request as AddForgeRequest +from swh.web.add_forge_now.models import RequestHistory from swh.web.auth.utils import is_add_forge_now_moderator def add_forge_request_list_datatables(request: HttpRequest) -> HttpResponse: """Dedicated endpoint used by datatables to display the add-forge requests in the Web UI. """ draw = int(request.GET.get("draw", 0)) add_forge_requests = AddForgeRequest.objects.all() table_data: Dict[str, Any] = { "recordsTotal": add_forge_requests.count(), "draw": draw, } search_value = request.GET.get("search[value]") column_order = request.GET.get("order[0][column]") field_order = request.GET.get(f"columns[{column_order}][name]", "id") order_dir = request.GET.get("order[0][dir]", "desc") if field_order: if order_dir == "desc": field_order = "-" + field_order add_forge_requests = add_forge_requests.order_by(field_order) per_page = int(request.GET.get("length", 10)) page_num = int(request.GET.get("start", 0)) // per_page + 1 if search_value: add_forge_requests = add_forge_requests.filter( Q(forge_type__icontains=search_value) | Q(forge_url__icontains=search_value) | Q(status__icontains=search_value) ) if ( int(request.GET.get("user_requests_only", "0")) and request.user.is_authenticated ): add_forge_requests = add_forge_requests.filter( submitter_name=request.user.username ) paginator = Paginator(add_forge_requests, per_page) page = paginator.page(page_num) if is_add_forge_now_moderator(request.user): requests = AddForgeNowRequestSerializer(page.object_list, many=True).data else: requests = AddForgeNowRequestPublicSerializer(page.object_list, many=True).data results = [dict(req) for req in requests] table_data["recordsFiltered"] = add_forge_requests.count() table_data["data"] = results return JsonResponse(table_data) FORGE_TYPES: List[str] = [ "bitbucket", "cgit", "gitlab", "gitea", "heptapod", ] def create_request_create(request): """View to create a new 'add_forge_now' request.""" return render( request, - "add_forge_now/creation_form.html", + "add-forge-creation-form.html", {"forge_types": FORGE_TYPES}, ) def create_request_list(request): """View to list existing 'add_forge_now' requests.""" return render( request, - "add_forge_now/list.html", + "add-forge-list.html", ) def create_request_help(request): """View to explain 'add_forge_now'.""" return render( request, - "add_forge_now/help.html", + "add-forge-help.html", ) @user_passes_test( is_add_forge_now_moderator, redirect_field_name="next_path", login_url=settings.LOGIN_URL, ) def create_request_message_source(request: HttpRequest, id: int) -> HttpResponse: """View to retrieve the message source for a given request history entry""" try: history_entry = RequestHistory.objects.select_related("request").get( pk=id, message_source__isnull=False ) assert history_entry.message_source is not None except RequestHistory.DoesNotExist: return HttpResponse(status=404) response = HttpResponse( bytes(history_entry.message_source), content_type="text/email" ) filename = f"add-forge-now-{history_entry.request.forge_domain}-message{id}.eml" response["Content-Disposition"] = f'attachment; filename="{filename}"' return response - - -urlpatterns = [ - url( - r"^add-forge/request/list/datatables/$", - add_forge_request_list_datatables, - name="add-forge-request-list-datatables", - ), - url(r"^add-forge/request/create/$", create_request_create, name="forge-add-create"), - url(r"^add-forge/request/list/$", create_request_list, name="forge-add-list"), - url( - r"^add-forge/request/message-source/(?P\d+)/$", - create_request_message_source, - name="forge-add-message-source", - ), - url(r"^add-forge/request/help/$", create_request_help, name="forge-add-help"), -] diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py index 4ef703a1..43f9471f 100644 --- a/swh/web/admin/urls.py +++ b/swh/web/admin/urls.py @@ -1,27 +1,23 @@ # Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.contrib.auth.views import LoginView from django.shortcuts import redirect from django.urls import re_path as url from swh.web.admin.adminurls import AdminUrls import swh.web.admin.deposit # noqa -from swh.web.config import is_feature_enabled - -if is_feature_enabled("add_forge_now"): - import swh.web.admin.add_forge_now # noqa def _admin_default_view(request): return redirect("admin-origin-save-requests") urlpatterns = [ url(r"^$", _admin_default_view, name="admin"), url(r"^login/$", LoginView.as_view(template_name="login.html"), name="login"), ] urlpatterns += AdminUrls.get_url_patterns() diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py index 04297017..9fe9f56f 100644 --- a/swh/web/api/urls.py +++ b/swh/web/api/urls.py @@ -1,23 +1,22 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.api.apiurls import APIUrls -import swh.web.api.views.add_forge_now # noqa import swh.web.api.views.content # noqa import swh.web.api.views.directory # noqa import swh.web.api.views.graph # noqa import swh.web.api.views.identifiers # noqa import swh.web.api.views.metadata # noqa import swh.web.api.views.origin # noqa import swh.web.api.views.ping # noqa import swh.web.api.views.raw # noqa import swh.web.api.views.release # noqa import swh.web.api.views.revision # noqa import swh.web.api.views.snapshot # noqa import swh.web.api.views.stat # noqa import swh.web.api.views.vault # noqa urlpatterns = APIUrls.get_url_patterns() diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py index 0e40bec1..828900f7 100644 --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -1,524 +1,523 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime, timezone import functools import os import re from typing import Any, Callable, Dict, List, Mapping, Optional import urllib.parse from bs4 import BeautifulSoup from docutils.core import publish_parts import docutils.parsers.rst import docutils.utils from docutils.writers.html5_polyglot import HTMLTranslator, Writer from iso8601 import ParseError, parse_date from pkg_resources import get_distribution from prometheus_client.registry import CollectorRegistry import requests from requests.auth import HTTPBasicAuth from django.conf import settings from django.core.cache import cache from django.core.cache.backends.base import DEFAULT_TIMEOUT from django.http import HttpRequest, QueryDict from django.shortcuts import redirect from django.urls import resolve from django.urls import reverse as django_reverse from swh.web.auth.utils import ( ADD_FORGE_MODERATOR_PERMISSION, ADMIN_LIST_DEPOSIT_PERMISSION, MAILMAP_ADMIN_PERMISSION, ) from swh.web.common.exc import BadInputExc, sentry_capture_exception from swh.web.config import SWH_WEB_SERVER_NAME, get_config, search SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" swh_object_icons = { "alias": "mdi mdi-star", "branch": "mdi mdi-source-branch", "branches": "mdi mdi-source-branch", "content": "mdi mdi-file-document", "cnt": "mdi mdi-file-document", "directory": "mdi mdi-folder", "dir": "mdi mdi-folder", "origin": "mdi mdi-source-repository", "ori": "mdi mdi-source-repository", "person": "mdi mdi-account", "revisions history": "mdi mdi-history", "release": "mdi mdi-tag", "rel": "mdi mdi-tag", "releases": "mdi mdi-tag", "revision": "mdi mdi-rotate-90 mdi-source-commit", "rev": "mdi mdi-rotate-90 mdi-source-commit", "snapshot": "mdi mdi-camera", "snp": "mdi mdi-camera", "visits": "mdi mdi-calendar-month", } def reverse( viewname: str, url_args: Optional[Dict[str, Any]] = None, query_params: Optional[Mapping[str, Optional[str]]] = None, current_app: Optional[str] = None, urlconf: Optional[str] = None, request: Optional[HttpRequest] = None, ) -> str: """An override of django reverse function supporting query parameters. Args: viewname: the name of the django view from which to compute a url url_args: dictionary of url arguments indexed by their names query_params: dictionary of query parameters to append to the reversed url current_app: the name of the django app tighten to the view urlconf: url configuration module request: build an absolute URI if provided Returns: str: the url of the requested view with processed arguments and query parameters """ if url_args: url_args = {k: v for k, v in url_args.items() if v is not None} url = django_reverse( viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app ) params: Dict[str, str] = {} if query_params: params = {k: v for k, v in query_params.items() if v is not None} if params: query_dict = QueryDict("", mutable=True) query_dict.update(dict(sorted(params.items()))) url += "?" + query_dict.urlencode(safe="/;:") if request is not None: url = request.build_absolute_uri(url) return url def datetime_to_utc(date): """Returns datetime in UTC without timezone info Args: date (datetime.datetime): input datetime with timezone info Returns: datetime.datetime: datetime in UTC without timezone info """ if date.tzinfo and date.tzinfo != timezone.utc: return date.astimezone(tz=timezone.utc) else: return date def parse_iso8601_date_to_utc(iso_date: str) -> datetime: """Given an ISO 8601 datetime string, parse the result as UTC datetime. Returns: a timezone-aware datetime representing the parsed date Raises: swh.web.common.exc.BadInputExc: provided date does not respect ISO 8601 format Samples: - 2016-01-12 - 2016-01-12T09:19:12+0100 - 2007-01-14T20:34:22Z """ try: date = parse_date(iso_date) return datetime_to_utc(date) except ParseError as e: raise BadInputExc(e) def shorten_path(path): """Shorten the given path: for each hash present, only return the first 8 characters followed by an ellipsis""" sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}" sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}" ret = re.sub(sha256_re, r"\1...", path) return re.sub(sha1_re, r"\1...", ret) def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M:%S UTC"): """Turns a string representation of an ISO 8601 datetime string to UTC and format it into a more human readable one. For instance, from the following input string: '2017-05-04T13:27:13+02:00' the following one is returned: '04 May 2017, 11:27 UTC'. Custom format string may also be provided as parameter Args: iso_date (str): a string representation of an ISO 8601 date fmt (str): optional date formatting string Returns: str: a formatted string representation of the input iso date """ if not iso_date: return iso_date date = parse_iso8601_date_to_utc(iso_date) return date.strftime(fmt) def gen_path_info(path): """Function to generate path data navigation for use with a breadcrumb in the swh web ui. For instance, from a path /folder1/folder2/folder3, it returns the following list:: [{'name': 'folder1', 'path': 'folder1'}, {'name': 'folder2', 'path': 'folder1/folder2'}, {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] Args: path: a filesystem path Returns: list: a list of path data for navigation as illustrated above. """ path_info = [] if path: sub_paths = path.strip("/").split("/") path_from_root = "" for p in sub_paths: path_from_root += "/" + p path_info.append({"name": p, "path": path_from_root.strip("/")}) return path_info def parse_rst(text, report_level=2): """ Parse a reStructuredText string with docutils. Args: text (str): string with reStructuredText markups in it report_level (int): level of docutils report messages to print (1 info 2 warning 3 error 4 severe 5 none) Returns: docutils.nodes.document: a parsed docutils document """ parser = docutils.parsers.rst.Parser() components = (docutils.parsers.rst.Parser,) settings = docutils.frontend.OptionParser( components=components ).get_default_values() settings.report_level = report_level document = docutils.utils.new_document("rst-doc", settings=settings) parser.parse(text, document) return document def get_client_ip(request): """ Return the client IP address from an incoming HTTP request. Args: request (django.http.HttpRequest): the incoming HTTP request Returns: str: The client IP address """ x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") if x_forwarded_for: ip = x_forwarded_for.split(",")[0] else: ip = request.META.get("REMOTE_ADDR") return ip def is_swh_web_development(request: HttpRequest) -> bool: """Indicate if we are running a development version of swh-web.""" site_base_url = request.build_absolute_uri("/") return any( host in site_base_url for host in ("localhost", "127.0.0.1", "testserver") ) def is_swh_web_staging(request: HttpRequest) -> bool: """Indicate if we are running a staging version of swh-web.""" config = get_config() site_base_url = request.build_absolute_uri("/") return any( server_name in site_base_url for server_name in config["staging_server_names"] ) def is_swh_web_production(request: HttpRequest) -> bool: """Indicate if we are running the public production version of swh-web.""" return SWH_WEB_SERVER_NAME in request.build_absolute_uri("/") browsers_supported_image_mimes = set( [ "image/gif", "image/png", "image/jpeg", "image/bmp", "image/webp", "image/svg", "image/svg+xml", ] ) def context_processor(request): """ Django context processor used to inject variables in all swh-web templates. """ config = get_config() if ( hasattr(request, "user") and request.user.is_authenticated and not hasattr(request.user, "backend") ): # To avoid django.template.base.VariableDoesNotExist errors # when rendering templates when standard Django user is logged in. request.user.backend = "django.contrib.auth.backends.ModelBackend" return { "swh_object_icons": swh_object_icons, "available_languages": None, "swh_client_config": config["client_config"], "oidc_enabled": bool(config["keycloak"]["server_url"]), "browsers_supported_image_mimes": browsers_supported_image_mimes, "keycloak": config["keycloak"], "site_base_url": request.build_absolute_uri("/"), "DJANGO_SETTINGS_MODULE": os.environ["DJANGO_SETTINGS_MODULE"], "status": config["status"], "swh_web_dev": is_swh_web_development(request), "swh_web_staging": is_swh_web_staging(request), "swh_web_prod": is_swh_web_production(request), "swh_web_version": get_distribution("swh.web").version, "iframe_mode": False, "ADMIN_LIST_DEPOSIT_PERMISSION": ADMIN_LIST_DEPOSIT_PERMISSION, "ADD_FORGE_MODERATOR_PERMISSION": ADD_FORGE_MODERATOR_PERMISSION, - "FEATURES": get_config()["features"], "MAILMAP_ADMIN_PERMISSION": MAILMAP_ADMIN_PERMISSION, "lang": "en", "sidebar_state": request.COOKIES.get("sidebar-state", "expanded"), "SWH_DJANGO_APPS": settings.SWH_DJANGO_APPS, } def resolve_branch_alias( snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] ) -> Optional[Dict[str, Any]]: """ Resolve branch alias in snapshot content. Args: snapshot: a full snapshot content branch: a branch alias contained in the snapshot Returns: The real snapshot branch that got aliased. """ while branch and branch["target_type"] == "alias": if branch["target"] in snapshot["branches"]: branch = snapshot["branches"][branch["target"]] else: from swh.web.common import archive snp = archive.lookup_snapshot( snapshot["id"], branches_from=branch["target"], branches_count=1 ) if snp and branch["target"] in snp["branches"]: branch = snp["branches"][branch["target"]] else: branch = None return branch class _NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] _HTML_WRITER = Writer() _HTML_WRITER.translator_class = _NoHeaderHTMLTranslator def rst_to_html(rst: str) -> str: """ Convert reStructuredText document into HTML. Args: rst: A string containing a reStructuredText document Returns: Body content of the produced HTML conversion. """ settings = { "initial_header_level": 2, "halt_level": 4, "traceback": True, "file_insertion_enabled": False, "raw_enabled": False, } pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) return f'
{pp["html_body"]}
' def prettify_html(html: str) -> str: """ Prettify an HTML document. Args: html: Input HTML document Returns: The prettified HTML document """ return BeautifulSoup(html, "lxml").prettify() def django_cache( timeout: int = DEFAULT_TIMEOUT, catch_exception: bool = False, exception_return_value: Any = None, invalidate_cache_pred: Callable[[Any], bool] = lambda val: False, ): """Decorator to put the result of a function call in Django cache, subsequent calls will directly return the cached value. Args: timeout: The number of seconds value will be hold in cache catch_exception: If :const:`True`, any thrown exception by the decorated function will be caught and not reraised exception_return_value: The value to return if previous parameter is set to :const:`True` invalidate_cache_pred: A predicate function enabling to invalidate the cache under certain conditions, decorated function will then be called again Returns: The returned value of the decorated function for the specified parameters """ def inner(func): @functools.wraps(func) def wrapper(*args, **kwargs): func_args = args + (0,) + tuple(sorted(kwargs.items())) cache_key = str(hash((func.__module__, func.__name__) + func_args)) ret = cache.get(cache_key) if ret is None or invalidate_cache_pred(ret): try: ret = func(*args, **kwargs) except Exception as exc: if catch_exception: sentry_capture_exception(exc) return exception_return_value else: raise else: cache.set(cache_key, ret, timeout=timeout) return ret return wrapper return inner def _deposits_list_url( deposits_list_base_url: str, page_size: int, username: Optional[str] ) -> str: params = {"page_size": str(page_size)} if username is not None: params["username"] = username return f"{deposits_list_base_url}?{urllib.parse.urlencode(params)}" def get_deposits_list(username: Optional[str] = None) -> List[Dict[str, Any]]: """Return the list of software deposits using swh-deposit API""" config = get_config()["deposit"] private_api_url = config["private_api_url"].rstrip("/") + "/" deposits_list_base_url = private_api_url + "deposits" deposits_list_auth = HTTPBasicAuth( config["private_api_user"], config["private_api_password"] ) deposits_list_url = _deposits_list_url( deposits_list_base_url, page_size=1, username=username ) nb_deposits = requests.get( deposits_list_url, auth=deposits_list_auth, timeout=30 ).json()["count"] @django_cache(invalidate_cache_pred=lambda data: data["count"] != nb_deposits) def _get_deposits_data(): deposits_list_url = _deposits_list_url( deposits_list_base_url, page_size=nb_deposits, username=username ) return requests.get( deposits_list_url, auth=deposits_list_auth, timeout=30, ).json() deposits_data = _get_deposits_data() return deposits_data["results"] _origin_visit_types_cache_timeout = 24 * 60 * 60 # 24 hours @django_cache( timeout=_origin_visit_types_cache_timeout, catch_exception=True, exception_return_value=[], ) def origin_visit_types() -> List[str]: """Return the exhaustive list of visit types for origins ingested into the archive. """ return sorted(search().visit_types_count().keys()) def redirect_to_new_route(request, new_route, permanent=True): """Redirect a request to another route with url args and query parameters eg: /origin//log?path=test can be redirected as /log?url=&path=test. This can be used to deprecate routes """ request_path = resolve(request.path_info) args = {**request_path.kwargs, **request.GET.dict()} return redirect( reverse(new_route, query_params=args), permanent=permanent, ) diff --git a/swh/web/config.py b/swh/web/config.py index 111455eb..b763dc04 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,242 +1,234 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from typing import Any, Dict from swh.core import config from swh.counters import get_counters from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SWH_WEB_SERVER_NAME = "archive.softwareheritage.org" SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org" SWH_WEB_STAGING_SERVER_NAMES = [ "webapp.staging.swh.network", "webapp.internal.staging.swh.network", ] SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { "allowed_hosts": ("list", []), "storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10, }, ), "indexer_storage": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1, }, ), "counters": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5011/", "timeout": 1, }, ), "search": ( "dict", { "cls": "remote", "url": "http://127.0.0.1:5010/", "timeout": 10, }, ), "search_config": ( "dict", { "metadata_backend": "swh-indexer-storage", }, # or "swh-search" ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), "serve_assets": ("bool", False), "host": ("string", "127.0.0.1"), "port": ("int", 5004), "secret_key": ("string", "development key"), # do not display code highlighting for content > 1MB "content_display_max_size": ("int", 5 * 1024 * 1024), "snapshot_content_max_size": ("int", 1000), "throttling": ( "dict", { "cache_uri": None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None "scopes": { "swh_api": { "limiter_rate": {"default": "120/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_search": { "limiter_rate": {"default": "10/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_vault_cooking": { "limiter_rate": {"default": "120/h", "GET": "60/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_save_origin": { "limiter_rate": {"default": "120/h", "POST": "10/h"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_visit_latest": { "limiter_rate": {"default": "700/m"}, "exempted_networks": ["127.0.0.0/8"], }, }, }, ), "vault": ( "dict", { "cls": "remote", "args": { "url": "http://127.0.0.1:5005/", }, }, ), "scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}), "development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")), "test_db": ("dict", {"name": "swh-web-test"}), "production_db": ("dict", {"name": "swh-web"}), "deposit": ( "dict", { "private_api_url": "https://deposit.softwareheritage.org/1/private/", "private_api_user": "swhworker", "private_api_password": "some-password", }, ), "e2e_tests_mode": ("bool", False), "es_workers_index_url": ("string", ""), "history_counters_url": ( "string", ( "http://counters1.internal.softwareheritage.org:5011" "/counters_history/history.json" ), ), "client_config": ("dict", {}), "keycloak": ("dict", {"server_url": "", "realm_name": ""}), "graph": ( "dict", { "server_url": "http://graph.internal.softwareheritage.org:5009/graph/", "max_edges": {"staff": 0, "user": 100000, "anonymous": 1000}, }, ), "status": ( "dict", { "server_url": "https://status.softwareheritage.org/", "json_path": "1.0/status/578e5eddcdc0cc7951000520", }, ), "counters_backend": ("string", "swh-storage"), # or "swh-counters" "staging_server_names": ("list", SWH_WEB_STAGING_SERVER_NAMES), "instance_name": ("str", "archive-test.softwareheritage.org"), "give": ("dict", {"public_key": "", "token": ""}), "features": ("dict", {"add_forge_now": True}), "add_forge_now": ("dict", {"email_address": "add-forge-now@example.com"}), "swh_extra_django_apps": ( "list", [ "swh.web.inbound_email", "swh.web.add_forge_now", "swh.web.mailmap", "swh.web.save_code_now", ], ), } swhweb_config: Dict[str, Any] = {} def get_config(config_file="web/web"): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get("SWH_CONFIG_FILENAME") if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, "log_dir") if swhweb_config.get("search"): swhweb_config["search"] = get_search(**swhweb_config["search"]) else: swhweb_config["search"] = None swhweb_config["storage"] = get_storage(**swhweb_config["storage"]) swhweb_config["vault"] = get_vault(**swhweb_config["vault"]) swhweb_config["indexer_storage"] = get_indexer_storage( **swhweb_config["indexer_storage"] ) swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) swhweb_config["counters"] = get_counters(**swhweb_config["counters"]) return swhweb_config def search(): """Return the current application's search.""" return get_config()["search"] def storage(): """Return the current application's storage.""" return get_config()["storage"] def vault(): """Return the current application's vault.""" return get_config()["vault"] def indexer_storage(): """Return the current application's indexer storage.""" return get_config()["indexer_storage"] def scheduler(): """Return the current application's scheduler.""" return get_config()["scheduler"] def counters(): """Return the current application's counters.""" return get_config()["counters"] - - -def is_feature_enabled(feature_name: str) -> bool: - """Determine whether a feature is enabled or not. If feature_name is not found at all, - it's considered disabled. - - """ - return get_config()["features"].get(feature_name, False) diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html index 86f89492..8ae81521 100644 --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -1,315 +1,315 @@ {% comment %} Copyright (C) 2015-2022 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load js_reverse %} {% load static %} {% load render_bundle from webpack_loader %} {% load swh_templatetags %} {% block title %}{% endblock %} {% render_bundle 'vendors' %} {% render_bundle 'webapp' %} {% render_bundle 'guided_tour' %} {{ request.user.is_authenticated|json_script:"swh_user_logged_in" }} {% include "includes/favicon.html" %} {% block header %}{% endblock %} {% if swh_web_prod %} {% endif %}
{% include "misc/hiring-banner.html" %}
{% if swh_web_staging %}
Staging
v{{ swh_web_version }}
{% elif swh_web_dev %}
Development
v{{ swh_web_version|split:"+"|first }}
{% endif %} {% block content %}{% endblock %}
{% include "includes/global-modals.html" %}
back to top
diff --git a/swh/web/tests/add_forge_now/__init__.py b/swh/web/tests/add_forge_now/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/tests/api/views/test_add_forge_now.py b/swh/web/tests/add_forge_now/test_api_views.py similarity index 99% rename from swh/web/tests/api/views/test_add_forge_now.py rename to swh/web/tests/add_forge_now/test_api_views.py index 9f938111..5c0d1138 100644 --- a/swh/web/tests/api/views/test_add_forge_now.py +++ b/swh/web/tests/add_forge_now/test_api_views.py @@ -1,651 +1,651 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import datetime import threading import time from typing import Dict from urllib.parse import urlencode, urlparse import iso8601 import pytest from swh.web.add_forge_now.models import Request, RequestHistory from swh.web.common.utils import reverse from swh.web.config import get_config from swh.web.inbound_email.utils import get_address_for_pk from swh.web.tests.utils import ( check_api_get_responses, check_api_post_response, check_http_get_response, check_http_post_response, ) @pytest.mark.django_db def test_add_forge_request_create_anonymous_user(api_client): url = reverse("api-1-add-forge-request-create") check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db def test_add_forge_request_create_empty(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") resp = check_api_post_response(api_client, url, status_code=400) assert '"forge_type"' in resp.data["reason"] ADD_FORGE_DATA_FORGE1: Dict = { "forge_type": "gitlab", "forge_url": "https://gitlab.example.org", "forge_contact_email": "admin@gitlab.example.org", "forge_contact_name": "gitlab.example.org admin", "forge_contact_comment": "user marked as owner in forge members", "submitter_forward_username": True, } ADD_FORGE_DATA_FORGE2: Dict = { "forge_type": "gitea", "forge_url": "https://gitea.example.org", "forge_contact_email": "admin@gitea.example.org", "forge_contact_name": "gitea.example.org admin", "forge_contact_comment": "user marked as owner in forge members", "submitter_forward_username": True, } ADD_FORGE_DATA_FORGE3: Dict = { "forge_type": "heptapod", "forge_url": "https://heptapod.host/", "forge_contact_email": "admin@example.org", "forge_contact_name": "heptapod admin", "forge_contact_comment": "", # authorized empty or null comment "submitter_forward_username": False, } ADD_FORGE_DATA_FORGE4: Dict = { **ADD_FORGE_DATA_FORGE3, "forge_url": "https://heptapod2.host/", "submitter_forward_username": "on", } ADD_FORGE_DATA_FORGE5: Dict = { **ADD_FORGE_DATA_FORGE3, "forge_url": "https://heptapod3.host/", "submitter_forward_username": "off", } def inbound_email_for_pk(pk: int) -> str: """Check that the inbound email matches the one expected for the given pk""" base_address = get_config()["add_forge_now"]["email_address"] return get_address_for_pk( salt="swh_web_add_forge_now", base_address=base_address, pk=pk ) @pytest.mark.django_db(transaction=True, reset_sequences=True) @pytest.mark.parametrize( "add_forge_data", [ ADD_FORGE_DATA_FORGE1, ADD_FORGE_DATA_FORGE2, ADD_FORGE_DATA_FORGE3, ADD_FORGE_DATA_FORGE4, ], ) def test_add_forge_request_create_success_post( api_client, regular_user, add_forge_data ): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") date_before = datetime.datetime.now(tz=datetime.timezone.utc) resp = check_api_post_response( api_client, url, data=add_forge_data, status_code=201, ) date_after = datetime.datetime.now(tz=datetime.timezone.utc) consent = add_forge_data["submitter_forward_username"] # map the expected result with what's expectedly read from the db to ease comparison expected_consent_bool = consent == "on" if isinstance(consent, str) else consent assert resp.data == { **add_forge_data, "id": resp.data["id"], "status": "PENDING", "submission_date": resp.data["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "submitter_forward_username": expected_consent_bool, "last_moderator": resp.data["last_moderator"], "last_modified_date": resp.data["last_modified_date"], "inbound_email_address": inbound_email_for_pk(resp.data["id"]), "forge_domain": urlparse(add_forge_data["forge_url"]).netloc, } assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after request = Request.objects.all().last() assert request.forge_url == add_forge_data["forge_url"] assert request.submitter_name == regular_user.username @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_success_form_encoded(client, regular_user): client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") date_before = datetime.datetime.now(tz=datetime.timezone.utc) resp = check_http_post_response( client, url, request_content_type="application/x-www-form-urlencoded", data=urlencode(ADD_FORGE_DATA_FORGE1), status_code=201, ) date_after = datetime.datetime.now(tz=datetime.timezone.utc) assert resp.data == { **ADD_FORGE_DATA_FORGE1, "id": resp.data["id"], "status": "PENDING", "submission_date": resp.data["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "last_moderator": resp.data["last_moderator"], "last_modified_date": resp.data["last_modified_date"], "inbound_email_address": inbound_email_for_pk(1), "forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc, } assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after request = Request.objects.all()[0] assert request.forge_url == ADD_FORGE_DATA_FORGE1["forge_url"] assert request.submitter_name == regular_user.username @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_duplicate(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") check_api_post_response( api_client, url, data=ADD_FORGE_DATA_FORGE1, status_code=201, ) check_api_post_response( api_client, url, data=ADD_FORGE_DATA_FORGE1, status_code=409, ) requests = Request.objects.all() assert len(requests) == 1 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_invalid_forge_url(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") forge_data = copy.deepcopy(ADD_FORGE_DATA_FORGE1) forge_data["forge_url"] = "foo" resp = check_api_post_response( api_client, url, data=forge_data, status_code=400, ) assert resp.data == { "exception": "BadInputExc", "reason": '{"forge_url": ["Enter a valid URL."]}', } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_anonymous_user(api_client): url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_regular_user(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_non_existent(api_client, add_forge_moderator): api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=400) def create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE1): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") return check_api_post_response( api_client, url, data=data, status_code=201, ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_empty(api_client, regular_user, add_forge_moderator): create_add_forge_request(api_client, regular_user) api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=400) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_missing_field( api_client, regular_user, add_forge_moderator ): create_add_forge_request(api_client, regular_user) api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, data={}, status_code=400) check_api_post_response( api_client, url, data={"new_status": "REJECTED"}, status_code=400 ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update(api_client, regular_user, add_forge_moderator): create_add_forge_request(api_client, regular_user) api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response( api_client, url, data={"text": "updating request"}, status_code=200 ) check_api_post_response( api_client, url, data={"new_status": "REJECTED", "text": "request rejected"}, status_code=200, ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_invalid_new_status( api_client, regular_user, add_forge_moderator ): create_add_forge_request(api_client, regular_user) api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response( api_client, url, data={"new_status": "ACCEPTED", "text": "request accepted"}, status_code=400, ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_status_concurrent( api_client, regular_user, add_forge_moderator, mocker ): _block_while_testing = mocker.patch( - "swh.web.api.views.add_forge_now._block_while_testing" + "swh.web.add_forge_now.api_views._block_while_testing" ) _block_while_testing.side_effect = lambda: time.sleep(1) create_add_forge_request(api_client, regular_user) api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) worker_ended = False def worker(): nonlocal worker_ended check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) worker_ended = True # this thread will first modify the request status to WAITING_FOR_FEEDBACK thread = threading.Thread(target=worker) thread.start() # the other thread (slower) will attempt to modify the request status to REJECTED # but it will not be allowed as the first faster thread already modified it # and REJECTED state can not be reached from WAITING_FOR_FEEDBACK one time.sleep(0.5) check_api_post_response( api_client, url, data={"new_status": "REJECTED", "text": "request accepted"}, status_code=400, ) thread.join() assert worker_ended @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_anonymous(api_client, regular_user): url = reverse("api-1-add-forge-request-list") resp = check_api_get_responses(api_client, url, status_code=200) assert resp.data == [] create_add_forge_request(api_client, regular_user) resp = check_api_get_responses(api_client, url, status_code=200) add_forge_request = { "forge_url": ADD_FORGE_DATA_FORGE1["forge_url"], "forge_type": ADD_FORGE_DATA_FORGE1["forge_type"], "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "id": resp.data[0]["id"], } assert resp.data == [add_forge_request] create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2) resp = check_api_get_responses(api_client, url, status_code=200) other_forge_request = { "forge_url": ADD_FORGE_DATA_FORGE2["forge_url"], "forge_type": ADD_FORGE_DATA_FORGE2["forge_type"], "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "id": resp.data[0]["id"], } assert resp.data == [other_forge_request, add_forge_request] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_moderator( api_client, regular_user, add_forge_moderator ): url = reverse("api-1-add-forge-request-list") create_add_forge_request(api_client, regular_user) create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2) api_client.force_login(add_forge_moderator) resp = check_api_get_responses(api_client, url, status_code=200) add_forge_request = { **ADD_FORGE_DATA_FORGE1, "status": "PENDING", "submission_date": resp.data[1]["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "last_moderator": resp.data[1]["last_moderator"], "last_modified_date": resp.data[1]["last_modified_date"], "id": resp.data[1]["id"], "inbound_email_address": inbound_email_for_pk(resp.data[1]["id"]), "forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc, } other_forge_request = { **ADD_FORGE_DATA_FORGE2, "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "last_moderator": resp.data[0]["last_moderator"], "last_modified_date": resp.data[0]["last_modified_date"], "id": resp.data[0]["id"], "inbound_email_address": inbound_email_for_pk(resp.data[0]["id"]), "forge_domain": urlparse(ADD_FORGE_DATA_FORGE2["forge_url"]).netloc, } assert resp.data == [other_forge_request, add_forge_request] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_pagination( api_client, regular_user, api_request_factory ): create_add_forge_request(api_client, regular_user) create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2) url = reverse("api-1-add-forge-request-list", query_params={"per_page": 1}) resp = check_api_get_responses(api_client, url, 200) assert len(resp.data) == 1 request = api_request_factory.get(url) next_url = reverse( "api-1-add-forge-request-list", query_params={"page": 2, "per_page": 1}, request=request, ) assert resp["Link"] == f'<{next_url}>; rel="next"' resp = check_api_get_responses(api_client, next_url, 200) assert len(resp.data) == 1 prev_url = reverse( "api-1-add-forge-request-list", query_params={"page": 1, "per_page": 1}, request=request, ) assert resp["Link"] == f'<{prev_url}>; rel="previous"' @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_submitter_filtering( api_client, regular_user, regular_user2 ): create_add_forge_request(api_client, regular_user) create_add_forge_request(api_client, regular_user2, data=ADD_FORGE_DATA_FORGE2) api_client.force_login(regular_user) url = reverse( "api-1-add-forge-request-list", query_params={"user_requests_only": 1} ) resp = check_api_get_responses(api_client, url, status_code=200) assert len(resp.data) == 1 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get(api_client, regular_user, add_forge_moderator): resp = create_add_forge_request(api_client, regular_user) submission_date = resp.data["submission_date"] url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) api_client.force_login(add_forge_moderator) check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) api_client.logout() url = reverse("api-1-add-forge-request-get", url_args={"id": 1}) resp = check_api_get_responses(api_client, url, status_code=200) assert resp.data == { "request": { "forge_url": ADD_FORGE_DATA_FORGE1["forge_url"], "forge_type": ADD_FORGE_DATA_FORGE1["forge_type"], "id": 1, "status": "WAITING_FOR_FEEDBACK", "submission_date": submission_date, }, "history": [ { "id": 1, "actor_role": "SUBMITTER", "date": resp.data["history"][0]["date"], "new_status": "PENDING", }, { "id": 2, "actor_role": "MODERATOR", "date": resp.data["history"][1]["date"], "new_status": "WAITING_FOR_FEEDBACK", }, ], } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get_moderator(api_client, regular_user, add_forge_moderator): resp = create_add_forge_request(api_client, regular_user) submission_date = resp.data["submission_date"] url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) api_client.force_login(add_forge_moderator) check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) url = reverse("api-1-add-forge-request-get", url_args={"id": 1}) resp = check_api_get_responses(api_client, url, status_code=200) resp.data["history"] = [dict(history_item) for history_item in resp.data["history"]] assert resp.data == { "request": { **ADD_FORGE_DATA_FORGE1, "id": 1, "status": "WAITING_FOR_FEEDBACK", "submission_date": submission_date, "submitter_name": regular_user.username, "submitter_email": regular_user.email, "last_moderator": add_forge_moderator.username, "last_modified_date": resp.data["history"][1]["date"], "inbound_email_address": inbound_email_for_pk(1), "forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc, }, "history": [ { "id": 1, "text": "", "actor": regular_user.username, "actor_role": "SUBMITTER", "date": resp.data["history"][0]["date"], "new_status": "PENDING", "message_source_url": None, }, { "id": 2, "text": "waiting for message", "actor": add_forge_moderator.username, "actor_role": "MODERATOR", "date": resp.data["history"][1]["date"], "new_status": "WAITING_FOR_FEEDBACK", "message_source_url": None, }, ], } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get_moderator_message_source( api_client, regular_user, add_forge_moderator ): resp = create_add_forge_request(api_client, regular_user) rh = RequestHistory( request=Request.objects.get(pk=resp.data["id"]), new_status="WAITING_FOR_FEEDBACK", text="waiting for message", actor=add_forge_moderator.username, actor_role="MODERATOR", message_source=b"test with a message source", ) rh.save() api_client.force_login(add_forge_moderator) url = reverse("api-1-add-forge-request-get", url_args={"id": resp.data["id"]}) resp = check_api_get_responses(api_client, url, status_code=200) resp.data["history"] = [dict(history_item) for history_item in resp.data["history"]] # Check that the authentified moderator can't urlhack non-existent message sources assert resp.data["history"][0]["message_source_url"] is None empty_message_url = reverse( "forge-add-message-source", url_args={"id": resp.data["history"][0]["id"]} ) check_http_get_response(api_client, empty_message_url, status_code=404) # Check that the authentified moderator can't urlhack non-existent message sources non_existent_message_url = reverse( "forge-add-message-source", url_args={"id": 9001} ) check_http_get_response(api_client, non_existent_message_url, status_code=404) # Check that the authentified moderator can access the message source when the url is # given message_source_url = resp.data["history"][-1]["message_source_url"] assert message_source_url is not None message_source_resp = check_http_get_response( api_client, message_source_url, status_code=200, content_type="text/email" ) # Check that the message source shows up as an attachment assert message_source_resp.content == rh.message_source disposition = message_source_resp["Content-Disposition"] assert disposition.startswith("attachment; filename=") assert disposition.endswith('.eml"') # Check that a regular user can't access message sources api_client.force_login(regular_user) check_http_get_response(api_client, message_source_url, status_code=302) api_client.force_login(add_forge_moderator) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get_invalid(api_client): url = reverse("api-1-add-forge-request-get", url_args={"id": 3}) check_api_get_responses(api_client, url, status_code=400) diff --git a/swh/web/tests/add_forge_now/test_app.py b/swh/web/tests/add_forge_now/test_app.py new file mode 100644 index 00000000..47cc92d8 --- /dev/null +++ b/swh/web/tests/add_forge_now/test_app.py @@ -0,0 +1,33 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from django.urls import get_resolver + +from swh.web.add_forge_now.urls import urlpatterns +from swh.web.common.utils import reverse +from swh.web.tests.django_asserts import assert_not_contains +from swh.web.tests.utils import check_html_get_response + + +@pytest.mark.django_db +def test_add_forge_now_deactivate(client, staff_user, django_settings): + """Check Add forge now feature is deactivated when the swh.web.add_forge_now django + application is not in installed apps.""" + + django_settings.SWH_DJANGO_APPS = [ + app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.add_forge_now" + ] + + url = reverse("swh-web-homepage") + client.force_login(staff_user) + resp = check_html_get_response(client, url, status_code=200) + assert_not_contains(resp, "swh-add-forge-now-item") + assert_not_contains(resp, "swh-add-forge-now-moderation-item") + + add_forge_now_view_names = set(urlpattern.name for urlpattern in urlpatterns) + all_view_names = set(get_resolver().reverse_dict.keys()) + assert add_forge_now_view_names & all_view_names == set() diff --git a/swh/web/tests/add_forge_now/test_views.py b/swh/web/tests/add_forge_now/test_views.py index 88750eae..a6a8eb23 100644 --- a/swh/web/tests/add_forge_now/test_views.py +++ b/swh/web/tests/add_forge_now/test_views.py @@ -1,221 +1,222 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import pytest from swh.web.common.utils import reverse -from swh.web.tests.api.views.test_add_forge_now import create_add_forge_request from swh.web.tests.utils import check_http_get_response +from .test_api_views import create_add_forge_request + NB_FORGE_TYPE = 2 NB_FORGES_PER_TYPE = 20 def create_add_forge_requests(client, regular_user, regular_user2): requests = [] for i in range(NB_FORGES_PER_TYPE): request = { "forge_type": "gitlab", "forge_url": f"https://gitlab.example{i:02d}.org", "forge_contact_email": f"admin@gitlab.example{i:02d}.org", "forge_contact_name": f"gitlab.example{i:02d}.org admin", "forge_contact_comment": "user marked as owner in forge members", } requests.append( json.loads( create_add_forge_request( client, regular_user, data=request, ).content ) ) request = { "forge_type": "gitea", "forge_url": f"https://gitea.example{i:02d}.org", "forge_contact_email": f"admin@gitea.example{i:02d}.org", "forge_contact_name": f"gitea.example{i:02d}.org admin", "forge_contact_comment": "user marked as owner in forge members", } requests.append( json.loads( create_add_forge_request( client, regular_user2, data=request, ).content ) ) return requests @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_datatables_no_parameters( client, regular_user, regular_user2 ): create_add_forge_requests(client, regular_user, regular_user2) url = reverse("add-forge-request-list-datatables") resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) length = 10 assert data["draw"] == 0 assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == length # default ordering is by descending id assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1 assert "submitter_name" not in data["data"][0] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_datatables( client, regular_user, regular_user2, add_forge_moderator ): create_add_forge_requests(client, regular_user, regular_user2) length = 10 url = reverse( "add-forge-request-list-datatables", query_params={"draw": 1, "length": length, "start": 0}, ) client.force_login(regular_user) resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) assert data["draw"] == 1 assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == length # default ordering is by descending id assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1 assert "submitter_name" not in data["data"][0] client.force_login(add_forge_moderator) resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) assert data["draw"] == 1 assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == length # default ordering is by descending id assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1 assert "submitter_name" in data["data"][0] assert "last_moderator" in data["data"][0] assert "last_modified_date" in data["data"][0] return data @pytest.mark.django_db(transaction=True, reset_sequences=True) @pytest.mark.parametrize("order_field", ["forge_url", "last_modified_date"]) def test_add_forge_request_list_datatables_ordering( client, add_forge_moderator, admin_user, order_field ): requests = create_add_forge_requests(client, add_forge_moderator, admin_user) requests_sorted = list(sorted(requests, key=lambda d: d[order_field])) forge_urls_asc = [request[order_field] for request in requests_sorted] forge_urls_desc = list(reversed(forge_urls_asc)) length = 10 client.force_login(admin_user) for direction in ("asc", "desc"): for i in range(4): url = reverse( "add-forge-request-list-datatables", query_params={ "draw": 1, "length": length, "start": i * length, "order[0][column]": 2, "order[0][dir]": direction, "columns[2][name]": order_field, }, ) resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) assert data["draw"] == 1 assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == length page_forge_urls = [request[order_field] for request in data["data"]] if direction == "asc": expected_forge_urls = forge_urls_asc[i * length : (i + 1) * length] else: expected_forge_urls = forge_urls_desc[i * length : (i + 1) * length] assert page_forge_urls == expected_forge_urls @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_datatables_search(client, regular_user, regular_user2): create_add_forge_requests(client, regular_user, regular_user2) url = reverse( "add-forge-request-list-datatables", query_params={ "draw": 1, "length": NB_FORGES_PER_TYPE, "start": 0, "search[value]": "gitlab", }, ) client.force_login(regular_user) resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) assert data["draw"] == 1 assert data["recordsFiltered"] == NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == NB_FORGES_PER_TYPE page_forge_type = [request["forge_type"] for request in data["data"]] assert page_forge_type == ["gitlab"] * NB_FORGES_PER_TYPE @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_datatables_user_requests( client, regular_user, regular_user2 ): create_add_forge_requests(client, regular_user, regular_user2) url = reverse( "add-forge-request-list-datatables", query_params={ "draw": 1, "length": NB_FORGES_PER_TYPE * NB_FORGE_TYPE, "start": 0, "user_requests_only": 1, }, ) client.force_login(regular_user2) resp = check_http_get_response(client, url, status_code=200) data = json.loads(resp.content) assert data["draw"] == 1 assert data["recordsFiltered"] == NB_FORGES_PER_TYPE assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE assert len(data["data"]) == NB_FORGES_PER_TYPE page_forge_type = [request["forge_type"] for request in data["data"]] assert page_forge_type == ["gitea"] * NB_FORGES_PER_TYPE diff --git a/swh/web/tests/test_config.py b/swh/web/tests/test_config.py deleted file mode 100644 index 571b3be0..00000000 --- a/swh/web/tests/test_config.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (C) 2022 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import pytest - -from swh.web.config import get_config, is_feature_enabled - - -@pytest.mark.parametrize( - "feature_name", - ["inexistant-feature", "awesome-stuff"], -) -def test_is_feature_enabled(feature_name): - config = get_config() - # by default, feature non configured are considered disabled - assert is_feature_enabled(feature_name) is False - - for enabled in [True, False]: - # Let's configure the feature - config["features"] = {feature_name: enabled} - # and check its configuration is properly read - assert is_feature_enabled(feature_name) is enabled diff --git a/swh/web/urls.py b/swh/web/urls.py index c0bf333c..72687d73 100644 --- a/swh/web/urls.py +++ b/swh/web/urls.py @@ -1,89 +1,86 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from importlib.util import find_spec from django_js_reverse.views import urls_js from django.conf import settings from django.conf.urls import handler400, handler403, handler404, handler500, include from django.contrib.auth.views import LogoutView from django.contrib.staticfiles.views import serve from django.shortcuts import render from django.urls import re_path as url from django.views.generic.base import RedirectView from swh.web.browse.identifiers import swhid_browse from swh.web.common.exc import ( swh_handle400, swh_handle403, swh_handle404, swh_handle500, ) from swh.web.common.utils import origin_visit_types -from swh.web.config import get_config, is_feature_enabled +from swh.web.config import get_config swh_web_config = get_config() favicon_view = RedirectView.as_view( url="/static/img/icons/swh-logo-32x32.png", permanent=True ) def _default_view(request): return render(request, "homepage.html", {"visit_types": origin_visit_types()}) urlpatterns = [ url(r"^admin/", include("swh.web.admin.urls")), url(r"^favicon\.ico/$", favicon_view), url(r"^$", _default_view, name="swh-web-homepage"), url(r"^jsreverse/$", urls_js, name="js_reverse"), # keep legacy SWHID resolving URL with trailing slash for backward compatibility url( r"^(?P(swh|SWH):[0-9]+:[A-Za-z]+:[0-9A-Fa-f]+.*)/$", swhid_browse, name="browse-swhid-legacy", ), url( r"^(?P(swh|SWH):[0-9]+:[A-Za-z]+:[0-9A-Fa-f]+.*)$", swhid_browse, name="browse-swhid", ), url(r"^", include("swh.web.misc.urls")), url(r"^", include("swh.web.auth.views")), url(r"^logout/$", LogoutView.as_view(template_name="logout.html"), name="logout"), ] # Register URLs for each SWH Django application for app in settings.SWH_DJANGO_APPS: app_urls = app + ".urls" try: app_urls_spec = find_spec(app_urls) if app_urls_spec is not None: urlpatterns.append(url(r"^", include(app_urls))) except ModuleNotFoundError: assert False, f"Django application {app} not found !" -if is_feature_enabled("add_forge_now"): - urlpatterns += (url(r"^", include("swh.web.add_forge_now.views")),) - # allow to serve assets through django staticfiles # even if settings.DEBUG is False def insecure_serve(request, path, **kwargs): return serve(request, path, insecure=True, **kwargs) # enable to serve compressed assets through django development server if swh_web_config["serve_assets"]: static_pattern = r"^%s(?P.*)/$" % settings.STATIC_URL[1:] urlpatterns.append(url(static_pattern, insecure_serve)) handler400 = swh_handle400 # noqa handler403 = swh_handle403 # noqa handler404 = swh_handle404 # noqa handler500 = swh_handle500 # noqa