diff --git a/swh/web/admin/add_forge_now.py b/swh/web/add_forge_now/admin_views.py
similarity index 77%
rename from swh/web/admin/add_forge_now.py
rename to swh/web/add_forge_now/admin_views.py
index 64a1e260..bdaefbc7 100644
--- a/swh/web/admin/add_forge_now.py
+++ b/swh/web/add_forge_now/admin_views.py
@@ -1,44 +1,35 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.shortcuts import render
from swh.web.add_forge_now.models import RequestStatus
-from swh.web.admin.adminurls import admin_route
from swh.web.auth.utils import is_add_forge_now_moderator
-@admin_route(
- r"add-forge/requests/",
- view_name="add-forge-now-requests-moderation",
-)
@user_passes_test(is_add_forge_now_moderator, login_url=settings.LOGIN_URL)
def add_forge_now_requests_moderation_dashboard(request):
"""Moderation dashboard to allow listing current requests."""
return render(
request,
- "add_forge_now/requests-moderation.html",
+ "add-forge-requests-moderation.html",
{"heading": "Add forge now requests moderation"},
)
-@admin_route(
- r"add-forge/request/(?P<request_id>(\d)+)/",
- view_name="add-forge-now-request-dashboard",
-)
@user_passes_test(is_add_forge_now_moderator, login_url=settings.LOGIN_URL)
def add_forge_now_request_dashboard(request, request_id):
"""Moderation dashboard to allow listing current requests."""
return render(
request,
- "add_forge_now/request-dashboard.html",
+ "add-forge-request-dashboard.html",
{
"request_id": request_id,
"heading": "Add forge now request dashboard",
"next_statuses_for": RequestStatus.next_statuses_str(),
},
)
diff --git a/swh/web/api/views/add_forge_now.py b/swh/web/add_forge_now/api_views.py
similarity index 100%
rename from swh/web/api/views/add_forge_now.py
rename to swh/web/add_forge_now/api_views.py
diff --git a/swh/web/templates/add_forge_now/common.html b/swh/web/add_forge_now/templates/add-forge-common.html
similarity index 98%
rename from swh/web/templates/add_forge_now/common.html
rename to swh/web/add_forge_now/templates/add-forge-common.html
index 24d34466..fed2359d 100644
--- a/swh/web/templates/add_forge_now/common.html
+++ b/swh/web/add_forge_now/templates/add-forge-common.html
@@ -1,72 +1,72 @@
-{% extends "../layout.html" %}
+{% extends "layout.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load render_bundle from webpack_loader %}
{% load static %}
{% block header %}
{% render_bundle 'add_forge' %}
{% endblock %}
{% block title %}
Add forge now – Software Heritage archive
{% endblock %}
{% block navbar-content %}
<h4>Request the addition of a forge into the archive</h4>
{% endblock %}
{% block content %}
<div class="row mt-3">
<p>
“Add forge now” provides a service for Software Heritage users to save a
complete forge in the Software Heritage archive by requesting the addition
of the forge URL into the list of regularly visited forges.
</p>
{% if not user.is_authenticated %}
<p>
You can submit an “Add forge now” request only when you are authenticated,
please login to submit the request.
</p>
{% endif %}
</div>
<!-- Tabs in the page -->
<ul class="nav nav-tabs">
<li class="nav-item">
<a class="nav-link {% if request.resolver_match.view_name == 'forge-add-create' %}active{% endif %}"
href="{% url 'forge-add-create' %}" id="swh-add-forge-tab">
Submit a Request
</a>
</li>
<li class="nav-item">
<a class="nav-link {% if request.resolver_match.view_name == 'forge-add-list' %}active{% endif %}"
href="{% url 'forge-add-list' %}" id="swh-add-forge-requests-list-tab">
Browse Requests
</a>
</li>
<li class="nav-item">
<a class="nav-link {% if request.resolver_match.view_name == 'forge-add-help' %}active{% endif %}"
href="{% url 'forge-add-help' %}" id="swh-add-forge-requests-help-tab">
Help
</a>
</li>
</ul>
<div class="tab-content">
{% block tab_content %}
{% endblock %}
</div>
<script>
swh.webapp.initPage('add-forge-now');
swh.add_forge.onCreateRequestPageLoad();
</script>
{% endblock %}
diff --git a/swh/web/templates/add_forge_now/creation_form.html b/swh/web/add_forge_now/templates/add-forge-creation-form.html
similarity index 99%
rename from swh/web/templates/add_forge_now/creation_form.html
rename to swh/web/add_forge_now/templates/add-forge-creation-form.html
index 6be8f222..86e65b2c 100644
--- a/swh/web/templates/add_forge_now/creation_form.html
+++ b/swh/web/add_forge_now/templates/add-forge-creation-form.html
@@ -1,129 +1,129 @@
-{% extends "./common.html" %}
+{% extends "./add-forge-common.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% block tab_content %}
<div id="swh-add-forge-submit-request" class="tab-pane active mt-3">
{% if not user.is_authenticated %}
<p class="text-primary">
You must be logged in to submit an add forge request. Please
<a id="loginLink" class="link-primary"
{% if oidc_enabled and 'remote_user' in request.GET %}
href="{% url 'oidc-login' %}?next={% url 'forge-add-create' %}"
{% else %}
href="{% url 'login' %}?next={% url 'forge-add-create' %}"
{% endif %}>log in</a>
</p>
{% else %}
<form method="POST" action="{% url 'api-1-add-forge-request-create' %}"
id="requestCreateForm" class="collapse show">
{% csrf_token %}
<div class="form-row">
<div class="form-group col-md-5">
<label for="swh-input-forge-type" class="swh-required-label">
Forge type
</label>
<select class="form-control" id="swh-input-forge-type"
name="forge_type" autofocus>
{% for forge_type in forge_types %}
<option value={{ forge_type }}>{{ forge_type}}</option>
{% endfor %}
</select>
<small class="form-text text-muted">
Supported forge types in software archive.
</small>
</div>
<div class="form-group col-md-7">
<label for="swh-input-forge-url" class="swh-required-label">
Forge URL
</label>
<input type="text" class="form-control" id="swh-input-forge-url"
name="forge_url" oninput="swh.add_forge.validateForgeUrl(this)" required>
<small class="form-text text-muted">
Remote URL of the forge.
</small>
</div>
</div>
<div class="form-row">
<div class="form-group col-md-5">
<label for="swh-input-forge-contact-name" class="swh-required-label">
Forge contact name
</label>
<input type="text" class="form-control" name="forge_contact_name"
id="swh-input-forge-contact-name" required>
<small class="form-text text-muted">
Name of the forge administrator.
</small>
</div>
<div class="form-group col-md-7">
<label for="swh-input-forge-contact-email" class="swh-required-label">
Forge contact email
</label>
<input type="email" class="form-control" name="forge_contact_email"
id="swh-input-forge-contact-email" required>
<small class="form-text text-muted">
Email of the forge administrator. The given email address will not be used
for any purpose outside the “add forge now” process.
</small>
</div>
</div>
<div class="form-row">
<div class="form-group form-check">
<input class="form-check-input" type="checkbox"
id="swh-input-consent-check" name="submitter_forward_username">
<label for="swh-input-consent-check">
I consent to add my username in the communication with the forge.
</label>
</div>
</div>
<div class="form-row">
<div class="form-group col-md-12">
<label for="swh-input-forge-comment">Comment</label>
<textarea class="form-control" id="swh-input-forge-comment"
name="forge_contact_comment" rows="3">
</textarea>
<small class="form-text text-muted">
Optionally, leave a comment to the moderator regarding your request.
</small>
</div>
</div>
<div class="form-row">
<div class="col-md-12">
<input id="swh-input-form-submit" type="submit" value="Submit Add Request"
class="btn btn-default float-right">
</div>
</div>
<div class="form-row">
<div class="col-md-12">
<h3 class="text-center">
<span id="userMessage" class="badge"></span>
</h3>
<p class="text-center">
<span id="userMessageDetail"></span>
</p>
</div>
</div>
</form>
<p>
Once an add-forge-request is submitted, its status can be viewed in
the <a id="swh-show-forge-add-requests-list" href="#browse-requests">
submitted requests list</a>. This process involves a moderator approval and
might take a few days to handle (it primarily depends on the response
time from the forge).
</p>
{% endif %}
</div>
{% endblock %}
diff --git a/swh/web/templates/add_forge_now/help.html b/swh/web/add_forge_now/templates/add-forge-help.html
similarity index 98%
rename from swh/web/templates/add_forge_now/help.html
rename to swh/web/add_forge_now/templates/add-forge-help.html
index 9b4d7092..4f89a646 100644
--- a/swh/web/templates/add_forge_now/help.html
+++ b/swh/web/add_forge_now/templates/add-forge-help.html
@@ -1,89 +1,89 @@
-{% extends "./common.html" %}
+{% extends "./add-forge-common.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% block tab_content %}
<div id="swh-add-forge-requests-help" class="tab-pane active mt-3">
<p>
For submitting an "Add forge now" request", you have to provide the following details:
</p>
<ul>
<li><strong>Forge type:</strong>Type of the forge you would like to add.
Supported forge types in software heritage currently are:
<ul>
<li><code>cgit</code>, for <a href="https://git.zx2c4.com/cgit/">cgit</a> forges</li>
<li><code>gitea</code>, for <a href="https://gitea.io/en-us/">gitea</a> forges</li>
<li><code>gitlab</code>, for <a href="https://about.gitlab.com/install/">gitlab</a> forges</li>
<li><code>heptapod</code>, for <a href="https://heptapod.net/">heptapod</a> forges</li>
...
</ul>
</li>
<li><strong>Forge url:</strong>The URL of the selected forge. This must be unique.
</li>
<li><strong>Forge contact name:</strong>Contact name of the forge administrator
</li>
<li><strong>Forge contact email:</strong>Contact email of the forge administrator. An email
will be sent to the given address to notify about the request.
</li>
<li><strong>Consent checkbox:</strong> This checkbox's purpose is to know whether we can
explicitly mention the user's login within the email sent to the forge. If
not checked, the user's name won't be mentioned in the email at all.
</li>
<li><strong>Comment:</strong> (Optionally) For the user to mention something more about
their request to the add-forge-now moderator.
</li>
</ul>
<p class="mt-1">
Once submitted, your "add forge" request can be in one
of the following states
</p>
<ul>
<li>
<strong>Pending:</strong>
The request was submitted and is waiting for a moderator
to validate
</li>
<li>
<strong>Waiting for feedback:</strong>
The request was processed by a moderator and the forge was contacted.
</li>
<li>
<strong>Feedback to handle:</strong>
The forge has responded to the request and
there is feedback to handle for the request.
</li>
<li>
<strong>Accepted:</strong>
The request has been accepted.
</li>
<li>
<strong>Scheduled:</strong>
The requested forge listing has been scheduled.
</li>
<li>
<strong>First listing done:</strong>
The first listing of the requested forge has been completed
</li>
<li>
<strong>First origin loaded:</strong>
The first repositories (or origins) from the requested forge have been loaded and archived.
</li>
<li><strong>Rejected:</strong>The request is invalid. It is rejected by a moderator with an explanation.</li>
<li><strong>Unsuccessful:</strong>An issue has been encountered which currently prevents the full archival.</li>
<li><strong>Suspended:</strong>The forge listing is not supported yet.</li>
</ul>
</div>
{% endblock %}
diff --git a/swh/web/templates/add_forge_now/list.html b/swh/web/add_forge_now/templates/add-forge-list.html
similarity index 94%
rename from swh/web/templates/add_forge_now/list.html
rename to swh/web/add_forge_now/templates/add-forge-list.html
index eb33fa69..1987168f 100644
--- a/swh/web/templates/add_forge_now/list.html
+++ b/swh/web/add_forge_now/templates/add-forge-list.html
@@ -1,24 +1,24 @@
-{% extends "./common.html" %}
+{% extends "./add-forge-common.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% block tab_content %}
<div id="swh-add-forge-requests-list" class="tab-pane active mt-3" style="width: 100%;">
<table id="add-forge-request-browse" class="table swh-table swh-table-striped">
<thead>
<tr>
<th>Submission date</th>
<th>Forge type</th>
<th>Forge URL</th>
<th>Status</th>
</tr>
</thead>
</table>
<div id="add-forge-browse-request-error"></div>
</div>
{% endblock %}
diff --git a/swh/web/templates/add_forge_now/request-dashboard.html b/swh/web/add_forge_now/templates/add-forge-request-dashboard.html
similarity index 99%
rename from swh/web/templates/add_forge_now/request-dashboard.html
rename to swh/web/add_forge_now/templates/add-forge-request-dashboard.html
index c263b910..61987e38 100644
--- a/swh/web/templates/add_forge_now/request-dashboard.html
+++ b/swh/web/add_forge_now/templates/add-forge-request-dashboard.html
@@ -1,122 +1,122 @@
-{% extends "../layout.html" %}
+{% extends "layout.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load render_bundle from webpack_loader %}
{% load static %}
{% load swh_templatetags %}
{% block header %}
{% render_bundle 'add_forge' %}
{% endblock %}
{% block title %}{{heading}} – Software Heritage archive{% endblock %}
{% block navbar-content %}
<h4>Add forge now request dashboard</h4>
{% endblock %}
{% block content %}
<div class="container">
<div id="fetchError" class="row d-none">
<h3>Error fetching information about the request</h3>
</div>
<div class="row" id="requestDetails">
<div id="ForgeRequestActions" class="request-actions">
<div>
<div class="accordion" id="requestHistory">
</div>
</div>
<div>
<form method="POST"
action="{% url 'api-1-add-forge-request-update' request_id %}"
style="padding-top: 5px;" id="updateRequestForm">
{% csrf_token %}
<div class="form-row">
<div class="form-group col-md-6">
<label for="decisionOptions" class="swh-required-label">Choose your decision</label>
<select class="form-control" id="decisionOptions" name="new_status">
</select>
</div>
</div>
<div class="form-row">
<div class="form-group col-md-12">
<label for="swh-input-forge-comment" class="swh-required-label">Comment</label>
<textarea class="form-control" id="updateComment" name="text" rows="3" required></textarea>
<small class="form-text text-muted">
Enter a comment related to your decision.
</small>
</div>
</div>
<div class="form-group col-md-6">
<button type="submit" class="btn btn-default mb-2">Submit</button>
</div>
<div class="form-row">
<div class="col-md-12">
<h3 class="text-center">
<span id="userMessage" class="badge"></span>
</h3>
</div>
</div>
</form>
</div>
</div>
<div id="ForgeRequestDetails" class="col-md-2 details-pane">
<div>
<strong>Request status</strong>
</div>
<small class="details-text" id="requestStatus"></small>
<hr />
<div>
<strong>Forge type</strong>
</div>
<small class="details-text" id="requestType"></small>
<hr />
<div>
<strong>Forge URL</strong>
</div>
<small class="details-text" id="requestURL"></small>
<hr />
<div>
<strong>Contact name</strong>
</div>
<small class="details-text" id="requestContactName"></small>
<hr />
<div>
<strong>Consent to use name</strong>
</div>
<small class="details-text" id="requestContactConsent"></small>
<hr />
<div>
<strong>Contact email</strong>
</div>
<small class="details-text" id="requestContactEmail"></small>
<hr />
<div>
<strong>Message</strong>
</div>
<p><small id="submitterMessage" class="details-text"></small></p>
<hr />
<div>
<button class="btn btn-link" id="contactForgeAdmin"
emailSubject="" emailTo="" emailCc="">
Send message to the forge
</button>
</div>
</div>
</div>
</div>
<script>
swh.webapp.initPage('add-forge-now-moderation');
swh.add_forge.onRequestDashboardLoad("{{ request_id }}", {{ next_statuses_for | jsonify }});
</script>
{% endblock %}
diff --git a/swh/web/templates/add_forge_now/requests-moderation.html b/swh/web/add_forge_now/templates/add-forge-requests-moderation.html
similarity index 97%
rename from swh/web/templates/add_forge_now/requests-moderation.html
rename to swh/web/add_forge_now/templates/add-forge-requests-moderation.html
index 6d3010d1..dcdbe7b1 100644
--- a/swh/web/templates/add_forge_now/requests-moderation.html
+++ b/swh/web/add_forge_now/templates/add-forge-requests-moderation.html
@@ -1,48 +1,48 @@
-{% extends "../layout.html" %}
+{% extends "layout.html" %}
{% comment %}
Copyright (C) 2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load render_bundle from webpack_loader %}
{% load static %}
{% block header %}
{% render_bundle 'add_forge' %}
{% endblock %}
{% block title %}{{ heading }} – Software Heritage archive{% endblock %}
{% block navbar-content %}
<h4>Add forge now moderation</h4>
{% endblock %}
{% block content %}
<div class="row">
<div id="swh-add-forge-requests-list" style="width: 100%;">
<table id="swh-add-forge-now-moderation-list" class="table swh-table swh-table-striped">
<thead>
<tr>
<th>ID</th>
<th>Submission date</th>
<th>Forge type</th>
<th>Forge URL</th>
<th>Moderator Name</th>
<th>Last Modified Date</th>
<th>Status</th>
</tr>
</thead>
</table>
<p id="swh-add-forge-now-moderation-list-error"></p>
</div>
</div>
<script>
swh.webapp.initPage('add-forge-now-moderation');
swh.add_forge.onModerationPageLoad();
</script>
{% endblock %}
diff --git a/swh/web/add_forge_now/urls.py b/swh/web/add_forge_now/urls.py
new file mode 100644
index 00000000..e8b1423e
--- /dev/null
+++ b/swh/web/add_forge_now/urls.py
@@ -0,0 +1,47 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from django.urls import re_path as url
+
+from swh.web.add_forge_now.admin_views import (
+ add_forge_now_request_dashboard,
+ add_forge_now_requests_moderation_dashboard,
+)
+
+# register Web API endpoints
+import swh.web.add_forge_now.api_views # noqa
+from swh.web.add_forge_now.views import (
+ add_forge_request_list_datatables,
+ create_request_create,
+ create_request_help,
+ create_request_list,
+ create_request_message_source,
+)
+
+urlpatterns = [
+ url(
+ r"^add-forge/request/list/datatables/$",
+ add_forge_request_list_datatables,
+ name="add-forge-request-list-datatables",
+ ),
+ url(r"^add-forge/request/create/$", create_request_create, name="forge-add-create"),
+ url(r"^add-forge/request/list/$", create_request_list, name="forge-add-list"),
+ url(
+ r"^add-forge/request/message-source/(?P<id>\d+)/$",
+ create_request_message_source,
+ name="forge-add-message-source",
+ ),
+ url(r"^add-forge/request/help/$", create_request_help, name="forge-add-help"),
+ url(
+ r"^admin/add-forge/requests/$",
+ add_forge_now_requests_moderation_dashboard,
+ name="add-forge-now-requests-moderation",
+ ),
+ url(
+ r"^admin/add-forge/request/(?P<request_id>(\d)+)/$",
+ add_forge_now_request_dashboard,
+ name="add-forge-now-request-dashboard",
+ ),
+]
diff --git a/swh/web/add_forge_now/views.py b/swh/web/add_forge_now/views.py
index c17fdb06..a2f41c10 100644
--- a/swh/web/add_forge_now/views.py
+++ b/swh/web/add_forge_now/views.py
@@ -1,160 +1,142 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from typing import Any, Dict, List
from django.conf import settings
from django.contrib.auth.decorators import user_passes_test
from django.core.paginator import Paginator
from django.db.models import Q
from django.http.request import HttpRequest
from django.http.response import HttpResponse, JsonResponse
from django.shortcuts import render
-from django.urls import re_path as url
-from swh.web.add_forge_now.models import Request as AddForgeRequest
-from swh.web.add_forge_now.models import RequestHistory
-from swh.web.api.views.add_forge_now import (
+from swh.web.add_forge_now.api_views import (
AddForgeNowRequestPublicSerializer,
AddForgeNowRequestSerializer,
)
+from swh.web.add_forge_now.models import Request as AddForgeRequest
+from swh.web.add_forge_now.models import RequestHistory
from swh.web.auth.utils import is_add_forge_now_moderator
def add_forge_request_list_datatables(request: HttpRequest) -> HttpResponse:
"""Dedicated endpoint used by datatables to display the add-forge
requests in the Web UI.
"""
draw = int(request.GET.get("draw", 0))
add_forge_requests = AddForgeRequest.objects.all()
table_data: Dict[str, Any] = {
"recordsTotal": add_forge_requests.count(),
"draw": draw,
}
search_value = request.GET.get("search[value]")
column_order = request.GET.get("order[0][column]")
field_order = request.GET.get(f"columns[{column_order}][name]", "id")
order_dir = request.GET.get("order[0][dir]", "desc")
if field_order:
if order_dir == "desc":
field_order = "-" + field_order
add_forge_requests = add_forge_requests.order_by(field_order)
per_page = int(request.GET.get("length", 10))
page_num = int(request.GET.get("start", 0)) // per_page + 1
if search_value:
add_forge_requests = add_forge_requests.filter(
Q(forge_type__icontains=search_value)
| Q(forge_url__icontains=search_value)
| Q(status__icontains=search_value)
)
if (
int(request.GET.get("user_requests_only", "0"))
and request.user.is_authenticated
):
add_forge_requests = add_forge_requests.filter(
submitter_name=request.user.username
)
paginator = Paginator(add_forge_requests, per_page)
page = paginator.page(page_num)
if is_add_forge_now_moderator(request.user):
requests = AddForgeNowRequestSerializer(page.object_list, many=True).data
else:
requests = AddForgeNowRequestPublicSerializer(page.object_list, many=True).data
results = [dict(req) for req in requests]
table_data["recordsFiltered"] = add_forge_requests.count()
table_data["data"] = results
return JsonResponse(table_data)
FORGE_TYPES: List[str] = [
"bitbucket",
"cgit",
"gitlab",
"gitea",
"heptapod",
]
def create_request_create(request):
"""View to create a new 'add_forge_now' request."""
return render(
request,
- "add_forge_now/creation_form.html",
+ "add-forge-creation-form.html",
{"forge_types": FORGE_TYPES},
)
def create_request_list(request):
"""View to list existing 'add_forge_now' requests."""
return render(
request,
- "add_forge_now/list.html",
+ "add-forge-list.html",
)
def create_request_help(request):
"""View to explain 'add_forge_now'."""
return render(
request,
- "add_forge_now/help.html",
+ "add-forge-help.html",
)
@user_passes_test(
is_add_forge_now_moderator,
redirect_field_name="next_path",
login_url=settings.LOGIN_URL,
)
def create_request_message_source(request: HttpRequest, id: int) -> HttpResponse:
"""View to retrieve the message source for a given request history entry"""
try:
history_entry = RequestHistory.objects.select_related("request").get(
pk=id, message_source__isnull=False
)
assert history_entry.message_source is not None
except RequestHistory.DoesNotExist:
return HttpResponse(status=404)
response = HttpResponse(
bytes(history_entry.message_source), content_type="text/email"
)
filename = f"add-forge-now-{history_entry.request.forge_domain}-message{id}.eml"
response["Content-Disposition"] = f'attachment; filename="{filename}"'
return response
-
-
-urlpatterns = [
- url(
- r"^add-forge/request/list/datatables/$",
- add_forge_request_list_datatables,
- name="add-forge-request-list-datatables",
- ),
- url(r"^add-forge/request/create/$", create_request_create, name="forge-add-create"),
- url(r"^add-forge/request/list/$", create_request_list, name="forge-add-list"),
- url(
- r"^add-forge/request/message-source/(?P<id>\d+)/$",
- create_request_message_source,
- name="forge-add-message-source",
- ),
- url(r"^add-forge/request/help/$", create_request_help, name="forge-add-help"),
-]
diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py
index 4ef703a1..43f9471f 100644
--- a/swh/web/admin/urls.py
+++ b/swh/web/admin/urls.py
@@ -1,27 +1,23 @@
# Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from django.contrib.auth.views import LoginView
from django.shortcuts import redirect
from django.urls import re_path as url
from swh.web.admin.adminurls import AdminUrls
import swh.web.admin.deposit # noqa
-from swh.web.config import is_feature_enabled
-
-if is_feature_enabled("add_forge_now"):
- import swh.web.admin.add_forge_now # noqa
def _admin_default_view(request):
return redirect("admin-origin-save-requests")
urlpatterns = [
url(r"^$", _admin_default_view, name="admin"),
url(r"^login/$", LoginView.as_view(template_name="login.html"), name="login"),
]
urlpatterns += AdminUrls.get_url_patterns()
diff --git a/swh/web/api/urls.py b/swh/web/api/urls.py
index 04297017..9fe9f56f 100644
--- a/swh/web/api/urls.py
+++ b/swh/web/api/urls.py
@@ -1,23 +1,22 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from swh.web.api.apiurls import APIUrls
-import swh.web.api.views.add_forge_now # noqa
import swh.web.api.views.content # noqa
import swh.web.api.views.directory # noqa
import swh.web.api.views.graph # noqa
import swh.web.api.views.identifiers # noqa
import swh.web.api.views.metadata # noqa
import swh.web.api.views.origin # noqa
import swh.web.api.views.ping # noqa
import swh.web.api.views.raw # noqa
import swh.web.api.views.release # noqa
import swh.web.api.views.revision # noqa
import swh.web.api.views.snapshot # noqa
import swh.web.api.views.stat # noqa
import swh.web.api.views.vault # noqa
urlpatterns = APIUrls.get_url_patterns()
diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py
index 0e40bec1..828900f7 100644
--- a/swh/web/common/utils.py
+++ b/swh/web/common/utils.py
@@ -1,524 +1,523 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from datetime import datetime, timezone
import functools
import os
import re
from typing import Any, Callable, Dict, List, Mapping, Optional
import urllib.parse
from bs4 import BeautifulSoup
from docutils.core import publish_parts
import docutils.parsers.rst
import docutils.utils
from docutils.writers.html5_polyglot import HTMLTranslator, Writer
from iso8601 import ParseError, parse_date
from pkg_resources import get_distribution
from prometheus_client.registry import CollectorRegistry
import requests
from requests.auth import HTTPBasicAuth
from django.conf import settings
from django.core.cache import cache
from django.core.cache.backends.base import DEFAULT_TIMEOUT
from django.http import HttpRequest, QueryDict
from django.shortcuts import redirect
from django.urls import resolve
from django.urls import reverse as django_reverse
from swh.web.auth.utils import (
ADD_FORGE_MODERATOR_PERMISSION,
ADMIN_LIST_DEPOSIT_PERMISSION,
MAILMAP_ADMIN_PERMISSION,
)
from swh.web.common.exc import BadInputExc, sentry_capture_exception
from swh.web.config import SWH_WEB_SERVER_NAME, get_config, search
SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True)
SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}"
swh_object_icons = {
"alias": "mdi mdi-star",
"branch": "mdi mdi-source-branch",
"branches": "mdi mdi-source-branch",
"content": "mdi mdi-file-document",
"cnt": "mdi mdi-file-document",
"directory": "mdi mdi-folder",
"dir": "mdi mdi-folder",
"origin": "mdi mdi-source-repository",
"ori": "mdi mdi-source-repository",
"person": "mdi mdi-account",
"revisions history": "mdi mdi-history",
"release": "mdi mdi-tag",
"rel": "mdi mdi-tag",
"releases": "mdi mdi-tag",
"revision": "mdi mdi-rotate-90 mdi-source-commit",
"rev": "mdi mdi-rotate-90 mdi-source-commit",
"snapshot": "mdi mdi-camera",
"snp": "mdi mdi-camera",
"visits": "mdi mdi-calendar-month",
}
def reverse(
viewname: str,
url_args: Optional[Dict[str, Any]] = None,
query_params: Optional[Mapping[str, Optional[str]]] = None,
current_app: Optional[str] = None,
urlconf: Optional[str] = None,
request: Optional[HttpRequest] = None,
) -> str:
"""An override of django reverse function supporting query parameters.
Args:
viewname: the name of the django view from which to compute a url
url_args: dictionary of url arguments indexed by their names
query_params: dictionary of query parameters to append to the
reversed url
current_app: the name of the django app tighten to the view
urlconf: url configuration module
request: build an absolute URI if provided
Returns:
str: the url of the requested view with processed arguments and
query parameters
"""
if url_args:
url_args = {k: v for k, v in url_args.items() if v is not None}
url = django_reverse(
viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app
)
params: Dict[str, str] = {}
if query_params:
params = {k: v for k, v in query_params.items() if v is not None}
if params:
query_dict = QueryDict("", mutable=True)
query_dict.update(dict(sorted(params.items())))
url += "?" + query_dict.urlencode(safe="/;:")
if request is not None:
url = request.build_absolute_uri(url)
return url
def datetime_to_utc(date):
"""Returns datetime in UTC without timezone info
Args:
date (datetime.datetime): input datetime with timezone info
Returns:
datetime.datetime: datetime in UTC without timezone info
"""
if date.tzinfo and date.tzinfo != timezone.utc:
return date.astimezone(tz=timezone.utc)
else:
return date
def parse_iso8601_date_to_utc(iso_date: str) -> datetime:
"""Given an ISO 8601 datetime string, parse the result as UTC datetime.
Returns:
a timezone-aware datetime representing the parsed date
Raises:
swh.web.common.exc.BadInputExc: provided date does not respect ISO 8601 format
Samples:
- 2016-01-12
- 2016-01-12T09:19:12+0100
- 2007-01-14T20:34:22Z
"""
try:
date = parse_date(iso_date)
return datetime_to_utc(date)
except ParseError as e:
raise BadInputExc(e)
def shorten_path(path):
"""Shorten the given path: for each hash present, only return the first
8 characters followed by an ellipsis"""
sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}"
sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}"
ret = re.sub(sha256_re, r"\1...", path)
return re.sub(sha1_re, r"\1...", ret)
def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M:%S UTC"):
"""Turns a string representation of an ISO 8601 datetime string
to UTC and format it into a more human readable one.
For instance, from the following input
string: '2017-05-04T13:27:13+02:00' the following one
is returned: '04 May 2017, 11:27 UTC'.
Custom format string may also be provided
as parameter
Args:
iso_date (str): a string representation of an ISO 8601 date
fmt (str): optional date formatting string
Returns:
str: a formatted string representation of the input iso date
"""
if not iso_date:
return iso_date
date = parse_iso8601_date_to_utc(iso_date)
return date.strftime(fmt)
def gen_path_info(path):
"""Function to generate path data navigation for use
with a breadcrumb in the swh web ui.
For instance, from a path /folder1/folder2/folder3,
it returns the following list::
[{'name': 'folder1', 'path': 'folder1'},
{'name': 'folder2', 'path': 'folder1/folder2'},
{'name': 'folder3', 'path': 'folder1/folder2/folder3'}]
Args:
path: a filesystem path
Returns:
list: a list of path data for navigation as illustrated above.
"""
path_info = []
if path:
sub_paths = path.strip("/").split("/")
path_from_root = ""
for p in sub_paths:
path_from_root += "/" + p
path_info.append({"name": p, "path": path_from_root.strip("/")})
return path_info
def parse_rst(text, report_level=2):
"""
Parse a reStructuredText string with docutils.
Args:
text (str): string with reStructuredText markups in it
report_level (int): level of docutils report messages to print
(1 info 2 warning 3 error 4 severe 5 none)
Returns:
docutils.nodes.document: a parsed docutils document
"""
parser = docutils.parsers.rst.Parser()
components = (docutils.parsers.rst.Parser,)
settings = docutils.frontend.OptionParser(
components=components
).get_default_values()
settings.report_level = report_level
document = docutils.utils.new_document("rst-doc", settings=settings)
parser.parse(text, document)
return document
def get_client_ip(request):
"""
Return the client IP address from an incoming HTTP request.
Args:
request (django.http.HttpRequest): the incoming HTTP request
Returns:
str: The client IP address
"""
x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR")
if x_forwarded_for:
ip = x_forwarded_for.split(",")[0]
else:
ip = request.META.get("REMOTE_ADDR")
return ip
def is_swh_web_development(request: HttpRequest) -> bool:
"""Indicate if we are running a development version of swh-web."""
site_base_url = request.build_absolute_uri("/")
return any(
host in site_base_url for host in ("localhost", "127.0.0.1", "testserver")
)
def is_swh_web_staging(request: HttpRequest) -> bool:
"""Indicate if we are running a staging version of swh-web."""
config = get_config()
site_base_url = request.build_absolute_uri("/")
return any(
server_name in site_base_url for server_name in config["staging_server_names"]
)
def is_swh_web_production(request: HttpRequest) -> bool:
"""Indicate if we are running the public production version of swh-web."""
return SWH_WEB_SERVER_NAME in request.build_absolute_uri("/")
browsers_supported_image_mimes = set(
[
"image/gif",
"image/png",
"image/jpeg",
"image/bmp",
"image/webp",
"image/svg",
"image/svg+xml",
]
)
def context_processor(request):
"""
Django context processor used to inject variables
in all swh-web templates.
"""
config = get_config()
if (
hasattr(request, "user")
and request.user.is_authenticated
and not hasattr(request.user, "backend")
):
# To avoid django.template.base.VariableDoesNotExist errors
# when rendering templates when standard Django user is logged in.
request.user.backend = "django.contrib.auth.backends.ModelBackend"
return {
"swh_object_icons": swh_object_icons,
"available_languages": None,
"swh_client_config": config["client_config"],
"oidc_enabled": bool(config["keycloak"]["server_url"]),
"browsers_supported_image_mimes": browsers_supported_image_mimes,
"keycloak": config["keycloak"],
"site_base_url": request.build_absolute_uri("/"),
"DJANGO_SETTINGS_MODULE": os.environ["DJANGO_SETTINGS_MODULE"],
"status": config["status"],
"swh_web_dev": is_swh_web_development(request),
"swh_web_staging": is_swh_web_staging(request),
"swh_web_prod": is_swh_web_production(request),
"swh_web_version": get_distribution("swh.web").version,
"iframe_mode": False,
"ADMIN_LIST_DEPOSIT_PERMISSION": ADMIN_LIST_DEPOSIT_PERMISSION,
"ADD_FORGE_MODERATOR_PERMISSION": ADD_FORGE_MODERATOR_PERMISSION,
- "FEATURES": get_config()["features"],
"MAILMAP_ADMIN_PERMISSION": MAILMAP_ADMIN_PERMISSION,
"lang": "en",
"sidebar_state": request.COOKIES.get("sidebar-state", "expanded"),
"SWH_DJANGO_APPS": settings.SWH_DJANGO_APPS,
}
def resolve_branch_alias(
snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]]
) -> Optional[Dict[str, Any]]:
"""
Resolve branch alias in snapshot content.
Args:
snapshot: a full snapshot content
branch: a branch alias contained in the snapshot
Returns:
The real snapshot branch that got aliased.
"""
while branch and branch["target_type"] == "alias":
if branch["target"] in snapshot["branches"]:
branch = snapshot["branches"][branch["target"]]
else:
from swh.web.common import archive
snp = archive.lookup_snapshot(
snapshot["id"], branches_from=branch["target"], branches_count=1
)
if snp and branch["target"] in snp["branches"]:
branch = snp["branches"][branch["target"]]
else:
branch = None
return branch
class _NoHeaderHTMLTranslator(HTMLTranslator):
"""
Docutils translator subclass to customize the generation of HTML
from reST-formatted docstrings
"""
def __init__(self, document):
super().__init__(document)
self.body_prefix = []
self.body_suffix = []
_HTML_WRITER = Writer()
_HTML_WRITER.translator_class = _NoHeaderHTMLTranslator
def rst_to_html(rst: str) -> str:
"""
Convert reStructuredText document into HTML.
Args:
rst: A string containing a reStructuredText document
Returns:
Body content of the produced HTML conversion.
"""
settings = {
"initial_header_level": 2,
"halt_level": 4,
"traceback": True,
"file_insertion_enabled": False,
"raw_enabled": False,
}
pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings)
return f'<div class="swh-rst">{pp["html_body"]}</div>'
def prettify_html(html: str) -> str:
"""
Prettify an HTML document.
Args:
html: Input HTML document
Returns:
The prettified HTML document
"""
return BeautifulSoup(html, "lxml").prettify()
def django_cache(
timeout: int = DEFAULT_TIMEOUT,
catch_exception: bool = False,
exception_return_value: Any = None,
invalidate_cache_pred: Callable[[Any], bool] = lambda val: False,
):
"""Decorator to put the result of a function call in Django cache,
subsequent calls will directly return the cached value.
Args:
timeout: The number of seconds value will be hold in cache
catch_exception: If :const:`True`, any thrown exception by
the decorated function will be caught and not reraised
exception_return_value: The value to return if previous
parameter is set to :const:`True`
invalidate_cache_pred: A predicate function enabling to
invalidate the cache under certain conditions, decorated
function will then be called again
Returns:
The returned value of the decorated function for the specified
parameters
"""
def inner(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
func_args = args + (0,) + tuple(sorted(kwargs.items()))
cache_key = str(hash((func.__module__, func.__name__) + func_args))
ret = cache.get(cache_key)
if ret is None or invalidate_cache_pred(ret):
try:
ret = func(*args, **kwargs)
except Exception as exc:
if catch_exception:
sentry_capture_exception(exc)
return exception_return_value
else:
raise
else:
cache.set(cache_key, ret, timeout=timeout)
return ret
return wrapper
return inner
def _deposits_list_url(
deposits_list_base_url: str, page_size: int, username: Optional[str]
) -> str:
params = {"page_size": str(page_size)}
if username is not None:
params["username"] = username
return f"{deposits_list_base_url}?{urllib.parse.urlencode(params)}"
def get_deposits_list(username: Optional[str] = None) -> List[Dict[str, Any]]:
"""Return the list of software deposits using swh-deposit API"""
config = get_config()["deposit"]
private_api_url = config["private_api_url"].rstrip("/") + "/"
deposits_list_base_url = private_api_url + "deposits"
deposits_list_auth = HTTPBasicAuth(
config["private_api_user"], config["private_api_password"]
)
deposits_list_url = _deposits_list_url(
deposits_list_base_url, page_size=1, username=username
)
nb_deposits = requests.get(
deposits_list_url, auth=deposits_list_auth, timeout=30
).json()["count"]
@django_cache(invalidate_cache_pred=lambda data: data["count"] != nb_deposits)
def _get_deposits_data():
deposits_list_url = _deposits_list_url(
deposits_list_base_url, page_size=nb_deposits, username=username
)
return requests.get(
deposits_list_url,
auth=deposits_list_auth,
timeout=30,
).json()
deposits_data = _get_deposits_data()
return deposits_data["results"]
_origin_visit_types_cache_timeout = 24 * 60 * 60 # 24 hours
@django_cache(
timeout=_origin_visit_types_cache_timeout,
catch_exception=True,
exception_return_value=[],
)
def origin_visit_types() -> List[str]:
"""Return the exhaustive list of visit types for origins
ingested into the archive.
"""
return sorted(search().visit_types_count().keys())
def redirect_to_new_route(request, new_route, permanent=True):
"""Redirect a request to another route with url args and query parameters
eg: /origin/<url:url-val>/log?path=test can be redirected as
/log?url=<url-val>&path=test. This can be used to deprecate routes
"""
request_path = resolve(request.path_info)
args = {**request_path.kwargs, **request.GET.dict()}
return redirect(
reverse(new_route, query_params=args),
permanent=permanent,
)
diff --git a/swh/web/config.py b/swh/web/config.py
index 111455eb..b763dc04 100644
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -1,242 +1,234 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
from typing import Any, Dict
from swh.core import config
from swh.counters import get_counters
from swh.indexer.storage import get_indexer_storage
from swh.scheduler import get_scheduler
from swh.search import get_search
from swh.storage import get_storage
from swh.vault import get_vault
from swh.web import settings
SWH_WEB_SERVER_NAME = "archive.softwareheritage.org"
SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org"
SWH_WEB_STAGING_SERVER_NAMES = [
"webapp.staging.swh.network",
"webapp.internal.staging.swh.network",
]
SETTINGS_DIR = os.path.dirname(settings.__file__)
DEFAULT_CONFIG = {
"allowed_hosts": ("list", []),
"storage": (
"dict",
{
"cls": "remote",
"url": "http://127.0.0.1:5002/",
"timeout": 10,
},
),
"indexer_storage": (
"dict",
{
"cls": "remote",
"url": "http://127.0.0.1:5007/",
"timeout": 1,
},
),
"counters": (
"dict",
{
"cls": "remote",
"url": "http://127.0.0.1:5011/",
"timeout": 1,
},
),
"search": (
"dict",
{
"cls": "remote",
"url": "http://127.0.0.1:5010/",
"timeout": 10,
},
),
"search_config": (
"dict",
{
"metadata_backend": "swh-indexer-storage",
}, # or "swh-search"
),
"log_dir": ("string", "/tmp/swh/log"),
"debug": ("bool", False),
"serve_assets": ("bool", False),
"host": ("string", "127.0.0.1"),
"port": ("int", 5004),
"secret_key": ("string", "development key"),
# do not display code highlighting for content > 1MB
"content_display_max_size": ("int", 5 * 1024 * 1024),
"snapshot_content_max_size": ("int", 1000),
"throttling": (
"dict",
{
"cache_uri": None, # production: memcached as cache (127.0.0.1:11211)
# development: in-memory cache so None
"scopes": {
"swh_api": {
"limiter_rate": {"default": "120/h"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_api_origin_search": {
"limiter_rate": {"default": "10/m"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_vault_cooking": {
"limiter_rate": {"default": "120/h", "GET": "60/m"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_save_origin": {
"limiter_rate": {"default": "120/h", "POST": "10/h"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_api_origin_visit_latest": {
"limiter_rate": {"default": "700/m"},
"exempted_networks": ["127.0.0.0/8"],
},
},
},
),
"vault": (
"dict",
{
"cls": "remote",
"args": {
"url": "http://127.0.0.1:5005/",
},
},
),
"scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}),
"development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")),
"test_db": ("dict", {"name": "swh-web-test"}),
"production_db": ("dict", {"name": "swh-web"}),
"deposit": (
"dict",
{
"private_api_url": "https://deposit.softwareheritage.org/1/private/",
"private_api_user": "swhworker",
"private_api_password": "some-password",
},
),
"e2e_tests_mode": ("bool", False),
"es_workers_index_url": ("string", ""),
"history_counters_url": (
"string",
(
"http://counters1.internal.softwareheritage.org:5011"
"/counters_history/history.json"
),
),
"client_config": ("dict", {}),
"keycloak": ("dict", {"server_url": "", "realm_name": ""}),
"graph": (
"dict",
{
"server_url": "http://graph.internal.softwareheritage.org:5009/graph/",
"max_edges": {"staff": 0, "user": 100000, "anonymous": 1000},
},
),
"status": (
"dict",
{
"server_url": "https://status.softwareheritage.org/",
"json_path": "1.0/status/578e5eddcdc0cc7951000520",
},
),
"counters_backend": ("string", "swh-storage"), # or "swh-counters"
"staging_server_names": ("list", SWH_WEB_STAGING_SERVER_NAMES),
"instance_name": ("str", "archive-test.softwareheritage.org"),
"give": ("dict", {"public_key": "", "token": ""}),
"features": ("dict", {"add_forge_now": True}),
"add_forge_now": ("dict", {"email_address": "add-forge-now@example.com"}),
"swh_extra_django_apps": (
"list",
[
"swh.web.inbound_email",
"swh.web.add_forge_now",
"swh.web.mailmap",
"swh.web.save_code_now",
],
),
}
swhweb_config: Dict[str, Any] = {}
def get_config(config_file="web/web"):
"""Read the configuration file `config_file`.
If an environment variable SWH_CONFIG_FILENAME is defined, this
takes precedence over the config_file parameter.
In any case, update the app with parameters (secret_key, conf)
and return the parsed configuration as a dict.
If no configuration file is provided, return a default
configuration.
"""
if not swhweb_config:
config_filename = os.environ.get("SWH_CONFIG_FILENAME")
if config_filename:
config_file = config_filename
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, "log_dir")
if swhweb_config.get("search"):
swhweb_config["search"] = get_search(**swhweb_config["search"])
else:
swhweb_config["search"] = None
swhweb_config["storage"] = get_storage(**swhweb_config["storage"])
swhweb_config["vault"] = get_vault(**swhweb_config["vault"])
swhweb_config["indexer_storage"] = get_indexer_storage(
**swhweb_config["indexer_storage"]
)
swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"])
swhweb_config["counters"] = get_counters(**swhweb_config["counters"])
return swhweb_config
def search():
"""Return the current application's search."""
return get_config()["search"]
def storage():
"""Return the current application's storage."""
return get_config()["storage"]
def vault():
"""Return the current application's vault."""
return get_config()["vault"]
def indexer_storage():
"""Return the current application's indexer storage."""
return get_config()["indexer_storage"]
def scheduler():
"""Return the current application's scheduler."""
return get_config()["scheduler"]
def counters():
"""Return the current application's counters."""
return get_config()["counters"]
-
-
-def is_feature_enabled(feature_name: str) -> bool:
- """Determine whether a feature is enabled or not. If feature_name is not found at all,
- it's considered disabled.
-
- """
- return get_config()["features"].get(feature_name, False)
diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html
index 86f89492..8ae81521 100644
--- a/swh/web/templates/layout.html
+++ b/swh/web/templates/layout.html
@@ -1,315 +1,315 @@
{% comment %}
Copyright (C) 2015-2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
<!DOCTYPE html>
{% load js_reverse %}
{% load static %}
{% load render_bundle from webpack_loader %}
{% load swh_templatetags %}
<html lang="en">
<head>
<meta charset="utf-8">
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<title>{% block title %}{% endblock %}</title>
{% render_bundle 'vendors' %}
{% render_bundle 'webapp' %}
{% render_bundle 'guided_tour' %}
<script>
/*
@licstart The following is the entire license notice for the JavaScript code in this page.
Copyright (C) 2015-2022 The Software Heritage developers
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
@licend The above is the entire license notice for the JavaScript code in this page.
*/
</script>
<script>
SWH_CONFIG = {{swh_client_config|jsonify}};
swh.webapp.sentryInit(SWH_CONFIG.sentry_dsn);
</script>
<script src="{% url 'js_reverse' %}" type="text/javascript"></script>
<script>
swh.webapp.setSwhObjectIcons({{ swh_object_icons|jsonify }});
</script>
{{ request.user.is_authenticated|json_script:"swh_user_logged_in" }}
{% include "includes/favicon.html" %}
{% block header %}{% endblock %}
{% if swh_web_prod %}
<!-- Matomo -->
<script type="text/javascript">
var _paq = window._paq = window._paq || [];
_paq.push(['trackPageView']);
(function() {
var u="https://piwik.inria.fr/";
_paq.push(['setTrackerUrl', u+'matomo.php']);
_paq.push(['setSiteId', '59']);
var d=document, g=d.createElement('script'), s=d.getElementsByTagName('script')[0];
g.type='text/javascript'; g.async=true; g.src=u+'matomo.js'; s.parentNode.insertBefore(g,s);
})();
</script>
<!-- End Matomo Code -->
{% endif %}
</head>
<body class="hold-transition layout-fixed sidebar-mini {% if sidebar_state == 'collapsed' %} sidebar-collapse {% endif %}">
<a id="top"></a>
<div class="wrapper">
<div class="swh-top-bar">
<ul>
<li class="swh-position-left">
<div id="swh-full-width-switch-container" class="custom-control custom-switch d-none d-lg-block d-xl-block">
<input type="checkbox" class="custom-control-input" id="swh-full-width-switch" onclick="swh.webapp.fullWidthToggled(event)">
<label class="custom-control-label font-weight-normal" for="swh-full-width-switch">Full width</label>
</div>
</li>
<li>
<a href="https://www.softwareheritage.org">Home</a>
</li>
<li>
<a href="https://forge.softwareheritage.org/">Development</a>
</li>
<li>
<a href="https://docs.softwareheritage.org/devel/">Documentation</a>
</li>
<li>
<a class="swh-donate-link" href="https://www.softwareheritage.org/donate">Donate</a>
</li>
<li class="swh-position-right">
<a href="{{ status.server_url }}" target="_blank"
class="swh-current-status mr-3 d-none d-lg-inline-block d-xl-inline-block">
<span id="swh-current-status-description">Operational</span>
<i class="swh-current-status-indicator green"></i>
</a>
{% url 'logout' as logout_url %}
{% if user.is_authenticated %}
Logged in as
{% if 'OIDC' in user.backend %}
<a id="swh-login" href="{% url 'oidc-profile' %}"><strong>{{ user.username }}</strong></a>,
<a href= "{% url 'oidc-logout' %}?next_path={% url 'logout' %}?remote_user=1">logout</a>
{% else %}
<strong id="swh-login">{{ user.username }}</strong>,
<a href="{{ logout_url }}">logout</a>
{% endif %}
{% elif oidc_enabled %}
{% if request.path != logout_url %}
<a id="swh-login" href="{% url 'oidc-login' %}?next_path={{ request.build_absolute_uri }}">login</a>
{% else %}
<a id="swh-login" href="{% url 'oidc-login' %}">login</a>
{% endif %}
{% else %}
{% if request.path != logout_url %}
<a id="swh-login" href="{% url 'login' %}?next={{ request.build_absolute_uri }}">login</a>
{% else %}
<a id="swh-login" href="{% url 'login' %}">login</a>
{% endif %}
{% endif %}
</li>
</ul>
</div>
<div class="swh-banner">
{% include "misc/hiring-banner.html" %}
</div>
<nav class="main-header navbar navbar-expand-lg navbar-light navbar-static-top" id="swh-navbar">
<div class="navbar-header">
<a class="nav-link swh-push-menu" data-widget="pushmenu" data-enable-remember="true" href="#">
<i class="mdi mdi-24px mdi-menu mdi-fw" aria-hidden="true"></i>
</a>
</div>
<div class="navbar" style="width: 94%;">
<div class="swh-navbar-content">
{% block navbar-content %}{% endblock %}
{% if request.resolver_match.url_name != 'swh-web-homepage' and request.resolver_match.url_name != 'browse-search' %}
<form class="form-horizontal d-none d-md-flex input-group swh-search-navbar needs-validation"
id="swh-origins-search-top">
<input class="form-control"
placeholder="Enter a SWHID to resolve or keyword(s) to search for in origin URLs"
type="text" id="swh-origins-search-top-input"
oninput="swh.webapp.validateSWHIDInput(this)" required/>
<div class="input-group-append">
<button class="btn btn-primary" type="submit">
<i class="swh-search-icon mdi mdi-24px mdi-magnify" aria-hidden="true"></i>
</button>
</div>
</form>
{% endif %}
</div>
</div>
</nav>
</div>
<aside class="swh-sidebar main-sidebar sidebar-no-expand sidebar-light-primary elevation-4 swh-sidebar-{{ sidebar_state }}">
<a href="{% url 'swh-web-homepage' %}" class="brand-link">
<img class="brand-image" src="{% static 'img/swh-logo.png' %}">
<div class="brand-text sitename" href="{% url 'swh-web-homepage' %}">
<span class="first-word">Software</span> <span class="second-word">Heritage</span>
</div>
</a>
<a href="/" class="swh-words-logo">
<div class="swh-words-logo-swh">
<span class="first-word">Software</span>
<span class="second-word">Heritage</span>
</div>
<span>Archive</span>
</a>
<div class="sidebar">
<nav class="mt-2">
<ul class="nav nav-pills nav-sidebar flex-column" data-widget="treeview" role="menu" data-accordion="false">
<li class="nav-header">Features</li>
<li class="nav-item swh-search-item" title="Search archived software">
<a href="{% url 'browse-search' %}" class="nav-link swh-search-link">
<i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-magnify"></i>
<p>Search</p>
</a>
</li>
<li class="nav-item swh-vault-item" title="Download archived software from the Vault">
<a href="{% url 'browse-vault' %}" class="nav-link swh-vault-link">
<i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-download"></i>
<p>Downloads</p>
</a>
</li>
{% if "swh.web.save_code_now" in SWH_DJANGO_APPS %}
<li class="nav-item swh-origin-save-item" title="Request the saving of a software origin into the archive">
<a href="{% url 'origin-save' %}" class="nav-link swh-origin-save-link">
<i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-camera"></i>
<p>Save code now</p>
</a>
</li>
{% endif %}
- {% if FEATURES.add_forge_now %}
+ {% if "swh.web.add_forge_now" in SWH_DJANGO_APPS %}
<li class="nav-item swh-add-forge-now-item" title="Request adding a new forge listing">
<a href="{% url 'forge-add-create' %}" class="nav-link swh-add-forge-now-link">
<i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-anvil"></i>
<p>Add forge now</p>
</a>
</li>
{% endif %}
<li class="nav-item swh-help-item" title="How to browse the archive ?">
<a href="#" class="nav-link swh-help-link" onclick="swh.guided_tour.guidedTourButtonClick(event)">
<i style="color: #e20026;" class="nav-icon mdi mdi-24px mdi-help-circle"></i>
<p>Help</p>
</a>
</li>
{% if user.is_authenticated %}
<li class="nav-header">Administration</li>
{% if "swh.web.save_code_now" in SWH_DJANGO_APPS and user.is_staff %}
<li class="nav-item swh-origin-save-admin-item" title="Save code now administration">
<a href="{% url 'admin-origin-save-requests' %}" class="nav-link swh-origin-save-admin-link">
<i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-camera"></i>
<p>Save code now</p>
</a>
</li>
{% endif %}
- {% if FEATURES.add_forge_now %}
+ {% if "swh.web.add_forge_now" in SWH_DJANGO_APPS %}
{% if user.is_staff or ADD_FORGE_MODERATOR_PERMISSION in user.get_all_permissions %}
<li class="nav-item swh-add-forge-now-moderation-item" title="Add forge now moderation">
<a href="{% url 'add-forge-now-requests-moderation' %}" class="nav-link swh-add-forge-now-moderation-link">
<i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-anvil"></i>
<p>Add forge now</p>
</a>
</li>
{% endif %}
{% endif %}
{% if user.is_staff or ADMIN_LIST_DEPOSIT_PERMISSION in user.get_all_permissions %}
<li class="nav-item swh-deposit-admin-item" title="Deposit administration">
<a href="{% url 'admin-deposit' %}" class="nav-link swh-deposit-admin-link">
<i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-folder-upload"></i>
<p>Deposit</p>
</a>
</li>
{% endif %}
{% if "swh.web.mailmap" in SWH_DJANGO_APPS and MAILMAP_ADMIN_PERMISSION in user.get_all_permissions %}
<li class="nav-item swh-mailmap-admin-item" title="Mailmap administration">
<a href="{% url 'admin-mailmap' %}" class="nav-link swh-mailmap-admin-link">
<i style="color: #fecd1b;" class="nav-icon mdi mdi-24px mdi-email"></i>
<p>Mailmap</p>
</a>
</li>
{% endif %}
{% endif %}
</ul>
</nav>
</div>
</aside>
<div class="content-wrapper">
<section class="content">
<div class="container" id="swh-web-content">
{% if swh_web_staging %}
<div class="swh-corner-ribbon">Staging<br/>v{{ swh_web_version }}</div>
{% elif swh_web_dev %}
<div class="swh-corner-ribbon">Development<br/>v{{ swh_web_version|split:"+"|first }}</div>
{% endif %}
{% block content %}{% endblock %}
</div>
</section>
</div>
{% include "includes/global-modals.html" %}
<footer class="footer">
<div class="container text-center">
<a href="https://www.softwareheritage.org">Software Heritage</a> —
Copyright (C) 2015–{% now "Y" %}, The Software Heritage developers.
License: <a href="https://www.gnu.org/licenses/agpl.html">GNU
AGPLv3+</a>. <br/> The source code of Software Heritage <em>itself</em>
is available on
our <a href="https://forge.softwareheritage.org/">development
forge</a>. <br/> The source code files <em>archived</em> by Software
Heritage are available under their own copyright and licenses. <br/>
<span class="link-color">Terms of use: </span>
<a href="https://www.softwareheritage.org/legal/bulk-access-terms-of-use/">Archive access</a>,
<a href="https://www.softwareheritage.org/legal/api-terms-of-use/">API</a>-
<a href="https://www.softwareheritage.org/contact/">Contact</a>-
<a href="{% url 'jslicenses' %}" rel="jslicense">JavaScript license information</a>-
<a href="{% url 'api-1-homepage' %}">Web API</a><br/>
{% if "production" not in DJANGO_SETTINGS_MODULE %}
swh-web v{{ swh_web_version }}
{% endif %}
</div>
</footer>
<div id="back-to-top">
<a href="#top"><img alt="back to top" src="{% static 'img/arrow-up-small.png' %}" /></a>
</div>
<script>
swh.webapp.setContainerFullWidth();
var statusServerURL = {{ status.server_url|jsonify }};
var statusJsonPath = {{ status.json_path|jsonify }};
swh.webapp.initStatusWidget(statusServerURL + statusJsonPath);
</script>
</body>
</html>
diff --git a/swh/web/tests/add_forge_now/__init__.py b/swh/web/tests/add_forge_now/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/swh/web/tests/api/views/test_add_forge_now.py b/swh/web/tests/add_forge_now/test_api_views.py
similarity index 99%
rename from swh/web/tests/api/views/test_add_forge_now.py
rename to swh/web/tests/add_forge_now/test_api_views.py
index 9f938111..5c0d1138 100644
--- a/swh/web/tests/api/views/test_add_forge_now.py
+++ b/swh/web/tests/add_forge_now/test_api_views.py
@@ -1,651 +1,651 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import copy
import datetime
import threading
import time
from typing import Dict
from urllib.parse import urlencode, urlparse
import iso8601
import pytest
from swh.web.add_forge_now.models import Request, RequestHistory
from swh.web.common.utils import reverse
from swh.web.config import get_config
from swh.web.inbound_email.utils import get_address_for_pk
from swh.web.tests.utils import (
check_api_get_responses,
check_api_post_response,
check_http_get_response,
check_http_post_response,
)
@pytest.mark.django_db
def test_add_forge_request_create_anonymous_user(api_client):
url = reverse("api-1-add-forge-request-create")
check_api_post_response(api_client, url, status_code=403)
@pytest.mark.django_db
def test_add_forge_request_create_empty(api_client, regular_user):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
resp = check_api_post_response(api_client, url, status_code=400)
assert '"forge_type"' in resp.data["reason"]
ADD_FORGE_DATA_FORGE1: Dict = {
"forge_type": "gitlab",
"forge_url": "https://gitlab.example.org",
"forge_contact_email": "admin@gitlab.example.org",
"forge_contact_name": "gitlab.example.org admin",
"forge_contact_comment": "user marked as owner in forge members",
"submitter_forward_username": True,
}
ADD_FORGE_DATA_FORGE2: Dict = {
"forge_type": "gitea",
"forge_url": "https://gitea.example.org",
"forge_contact_email": "admin@gitea.example.org",
"forge_contact_name": "gitea.example.org admin",
"forge_contact_comment": "user marked as owner in forge members",
"submitter_forward_username": True,
}
ADD_FORGE_DATA_FORGE3: Dict = {
"forge_type": "heptapod",
"forge_url": "https://heptapod.host/",
"forge_contact_email": "admin@example.org",
"forge_contact_name": "heptapod admin",
"forge_contact_comment": "", # authorized empty or null comment
"submitter_forward_username": False,
}
ADD_FORGE_DATA_FORGE4: Dict = {
**ADD_FORGE_DATA_FORGE3,
"forge_url": "https://heptapod2.host/",
"submitter_forward_username": "on",
}
ADD_FORGE_DATA_FORGE5: Dict = {
**ADD_FORGE_DATA_FORGE3,
"forge_url": "https://heptapod3.host/",
"submitter_forward_username": "off",
}
def inbound_email_for_pk(pk: int) -> str:
"""Check that the inbound email matches the one expected for the given pk"""
base_address = get_config()["add_forge_now"]["email_address"]
return get_address_for_pk(
salt="swh_web_add_forge_now", base_address=base_address, pk=pk
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
@pytest.mark.parametrize(
"add_forge_data",
[
ADD_FORGE_DATA_FORGE1,
ADD_FORGE_DATA_FORGE2,
ADD_FORGE_DATA_FORGE3,
ADD_FORGE_DATA_FORGE4,
],
)
def test_add_forge_request_create_success_post(
api_client, regular_user, add_forge_data
):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
date_before = datetime.datetime.now(tz=datetime.timezone.utc)
resp = check_api_post_response(
api_client,
url,
data=add_forge_data,
status_code=201,
)
date_after = datetime.datetime.now(tz=datetime.timezone.utc)
consent = add_forge_data["submitter_forward_username"]
# map the expected result with what's expectedly read from the db to ease comparison
expected_consent_bool = consent == "on" if isinstance(consent, str) else consent
assert resp.data == {
**add_forge_data,
"id": resp.data["id"],
"status": "PENDING",
"submission_date": resp.data["submission_date"],
"submitter_name": regular_user.username,
"submitter_email": regular_user.email,
"submitter_forward_username": expected_consent_bool,
"last_moderator": resp.data["last_moderator"],
"last_modified_date": resp.data["last_modified_date"],
"inbound_email_address": inbound_email_for_pk(resp.data["id"]),
"forge_domain": urlparse(add_forge_data["forge_url"]).netloc,
}
assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after
request = Request.objects.all().last()
assert request.forge_url == add_forge_data["forge_url"]
assert request.submitter_name == regular_user.username
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_create_success_form_encoded(client, regular_user):
client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
date_before = datetime.datetime.now(tz=datetime.timezone.utc)
resp = check_http_post_response(
client,
url,
request_content_type="application/x-www-form-urlencoded",
data=urlencode(ADD_FORGE_DATA_FORGE1),
status_code=201,
)
date_after = datetime.datetime.now(tz=datetime.timezone.utc)
assert resp.data == {
**ADD_FORGE_DATA_FORGE1,
"id": resp.data["id"],
"status": "PENDING",
"submission_date": resp.data["submission_date"],
"submitter_name": regular_user.username,
"submitter_email": regular_user.email,
"last_moderator": resp.data["last_moderator"],
"last_modified_date": resp.data["last_modified_date"],
"inbound_email_address": inbound_email_for_pk(1),
"forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc,
}
assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after
request = Request.objects.all()[0]
assert request.forge_url == ADD_FORGE_DATA_FORGE1["forge_url"]
assert request.submitter_name == regular_user.username
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_create_duplicate(api_client, regular_user):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
check_api_post_response(
api_client,
url,
data=ADD_FORGE_DATA_FORGE1,
status_code=201,
)
check_api_post_response(
api_client,
url,
data=ADD_FORGE_DATA_FORGE1,
status_code=409,
)
requests = Request.objects.all()
assert len(requests) == 1
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_create_invalid_forge_url(api_client, regular_user):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
forge_data = copy.deepcopy(ADD_FORGE_DATA_FORGE1)
forge_data["forge_url"] = "foo"
resp = check_api_post_response(
api_client,
url,
data=forge_data,
status_code=400,
)
assert resp.data == {
"exception": "BadInputExc",
"reason": '{"forge_url": ["Enter a valid URL."]}',
}
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_anonymous_user(api_client):
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(api_client, url, status_code=403)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_regular_user(api_client, regular_user):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(api_client, url, status_code=403)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_non_existent(api_client, add_forge_moderator):
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(api_client, url, status_code=400)
def create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE1):
api_client.force_login(regular_user)
url = reverse("api-1-add-forge-request-create")
return check_api_post_response(
api_client,
url,
data=data,
status_code=201,
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_empty(api_client, regular_user, add_forge_moderator):
create_add_forge_request(api_client, regular_user)
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(api_client, url, status_code=400)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_missing_field(
api_client, regular_user, add_forge_moderator
):
create_add_forge_request(api_client, regular_user)
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(api_client, url, data={}, status_code=400)
check_api_post_response(
api_client, url, data={"new_status": "REJECTED"}, status_code=400
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update(api_client, regular_user, add_forge_moderator):
create_add_forge_request(api_client, regular_user)
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(
api_client, url, data={"text": "updating request"}, status_code=200
)
check_api_post_response(
api_client,
url,
data={"new_status": "REJECTED", "text": "request rejected"},
status_code=200,
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_invalid_new_status(
api_client, regular_user, add_forge_moderator
):
create_add_forge_request(api_client, regular_user)
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
check_api_post_response(
api_client,
url,
data={"new_status": "ACCEPTED", "text": "request accepted"},
status_code=400,
)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_update_status_concurrent(
api_client, regular_user, add_forge_moderator, mocker
):
_block_while_testing = mocker.patch(
- "swh.web.api.views.add_forge_now._block_while_testing"
+ "swh.web.add_forge_now.api_views._block_while_testing"
)
_block_while_testing.side_effect = lambda: time.sleep(1)
create_add_forge_request(api_client, regular_user)
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
worker_ended = False
def worker():
nonlocal worker_ended
check_api_post_response(
api_client,
url,
data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"},
status_code=200,
)
worker_ended = True
# this thread will first modify the request status to WAITING_FOR_FEEDBACK
thread = threading.Thread(target=worker)
thread.start()
# the other thread (slower) will attempt to modify the request status to REJECTED
# but it will not be allowed as the first faster thread already modified it
# and REJECTED state can not be reached from WAITING_FOR_FEEDBACK one
time.sleep(0.5)
check_api_post_response(
api_client,
url,
data={"new_status": "REJECTED", "text": "request accepted"},
status_code=400,
)
thread.join()
assert worker_ended
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_anonymous(api_client, regular_user):
url = reverse("api-1-add-forge-request-list")
resp = check_api_get_responses(api_client, url, status_code=200)
assert resp.data == []
create_add_forge_request(api_client, regular_user)
resp = check_api_get_responses(api_client, url, status_code=200)
add_forge_request = {
"forge_url": ADD_FORGE_DATA_FORGE1["forge_url"],
"forge_type": ADD_FORGE_DATA_FORGE1["forge_type"],
"status": "PENDING",
"submission_date": resp.data[0]["submission_date"],
"id": resp.data[0]["id"],
}
assert resp.data == [add_forge_request]
create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2)
resp = check_api_get_responses(api_client, url, status_code=200)
other_forge_request = {
"forge_url": ADD_FORGE_DATA_FORGE2["forge_url"],
"forge_type": ADD_FORGE_DATA_FORGE2["forge_type"],
"status": "PENDING",
"submission_date": resp.data[0]["submission_date"],
"id": resp.data[0]["id"],
}
assert resp.data == [other_forge_request, add_forge_request]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_moderator(
api_client, regular_user, add_forge_moderator
):
url = reverse("api-1-add-forge-request-list")
create_add_forge_request(api_client, regular_user)
create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2)
api_client.force_login(add_forge_moderator)
resp = check_api_get_responses(api_client, url, status_code=200)
add_forge_request = {
**ADD_FORGE_DATA_FORGE1,
"status": "PENDING",
"submission_date": resp.data[1]["submission_date"],
"submitter_name": regular_user.username,
"submitter_email": regular_user.email,
"last_moderator": resp.data[1]["last_moderator"],
"last_modified_date": resp.data[1]["last_modified_date"],
"id": resp.data[1]["id"],
"inbound_email_address": inbound_email_for_pk(resp.data[1]["id"]),
"forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc,
}
other_forge_request = {
**ADD_FORGE_DATA_FORGE2,
"status": "PENDING",
"submission_date": resp.data[0]["submission_date"],
"submitter_name": regular_user.username,
"submitter_email": regular_user.email,
"last_moderator": resp.data[0]["last_moderator"],
"last_modified_date": resp.data[0]["last_modified_date"],
"id": resp.data[0]["id"],
"inbound_email_address": inbound_email_for_pk(resp.data[0]["id"]),
"forge_domain": urlparse(ADD_FORGE_DATA_FORGE2["forge_url"]).netloc,
}
assert resp.data == [other_forge_request, add_forge_request]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_pagination(
api_client, regular_user, api_request_factory
):
create_add_forge_request(api_client, regular_user)
create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA_FORGE2)
url = reverse("api-1-add-forge-request-list", query_params={"per_page": 1})
resp = check_api_get_responses(api_client, url, 200)
assert len(resp.data) == 1
request = api_request_factory.get(url)
next_url = reverse(
"api-1-add-forge-request-list",
query_params={"page": 2, "per_page": 1},
request=request,
)
assert resp["Link"] == f'<{next_url}>; rel="next"'
resp = check_api_get_responses(api_client, next_url, 200)
assert len(resp.data) == 1
prev_url = reverse(
"api-1-add-forge-request-list",
query_params={"page": 1, "per_page": 1},
request=request,
)
assert resp["Link"] == f'<{prev_url}>; rel="previous"'
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_submitter_filtering(
api_client, regular_user, regular_user2
):
create_add_forge_request(api_client, regular_user)
create_add_forge_request(api_client, regular_user2, data=ADD_FORGE_DATA_FORGE2)
api_client.force_login(regular_user)
url = reverse(
"api-1-add-forge-request-list", query_params={"user_requests_only": 1}
)
resp = check_api_get_responses(api_client, url, status_code=200)
assert len(resp.data) == 1
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_get(api_client, regular_user, add_forge_moderator):
resp = create_add_forge_request(api_client, regular_user)
submission_date = resp.data["submission_date"]
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
api_client.force_login(add_forge_moderator)
check_api_post_response(
api_client,
url,
data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"},
status_code=200,
)
api_client.logout()
url = reverse("api-1-add-forge-request-get", url_args={"id": 1})
resp = check_api_get_responses(api_client, url, status_code=200)
assert resp.data == {
"request": {
"forge_url": ADD_FORGE_DATA_FORGE1["forge_url"],
"forge_type": ADD_FORGE_DATA_FORGE1["forge_type"],
"id": 1,
"status": "WAITING_FOR_FEEDBACK",
"submission_date": submission_date,
},
"history": [
{
"id": 1,
"actor_role": "SUBMITTER",
"date": resp.data["history"][0]["date"],
"new_status": "PENDING",
},
{
"id": 2,
"actor_role": "MODERATOR",
"date": resp.data["history"][1]["date"],
"new_status": "WAITING_FOR_FEEDBACK",
},
],
}
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_get_moderator(api_client, regular_user, add_forge_moderator):
resp = create_add_forge_request(api_client, regular_user)
submission_date = resp.data["submission_date"]
url = reverse("api-1-add-forge-request-update", url_args={"id": 1})
api_client.force_login(add_forge_moderator)
check_api_post_response(
api_client,
url,
data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"},
status_code=200,
)
url = reverse("api-1-add-forge-request-get", url_args={"id": 1})
resp = check_api_get_responses(api_client, url, status_code=200)
resp.data["history"] = [dict(history_item) for history_item in resp.data["history"]]
assert resp.data == {
"request": {
**ADD_FORGE_DATA_FORGE1,
"id": 1,
"status": "WAITING_FOR_FEEDBACK",
"submission_date": submission_date,
"submitter_name": regular_user.username,
"submitter_email": regular_user.email,
"last_moderator": add_forge_moderator.username,
"last_modified_date": resp.data["history"][1]["date"],
"inbound_email_address": inbound_email_for_pk(1),
"forge_domain": urlparse(ADD_FORGE_DATA_FORGE1["forge_url"]).netloc,
},
"history": [
{
"id": 1,
"text": "",
"actor": regular_user.username,
"actor_role": "SUBMITTER",
"date": resp.data["history"][0]["date"],
"new_status": "PENDING",
"message_source_url": None,
},
{
"id": 2,
"text": "waiting for message",
"actor": add_forge_moderator.username,
"actor_role": "MODERATOR",
"date": resp.data["history"][1]["date"],
"new_status": "WAITING_FOR_FEEDBACK",
"message_source_url": None,
},
],
}
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_get_moderator_message_source(
api_client, regular_user, add_forge_moderator
):
resp = create_add_forge_request(api_client, regular_user)
rh = RequestHistory(
request=Request.objects.get(pk=resp.data["id"]),
new_status="WAITING_FOR_FEEDBACK",
text="waiting for message",
actor=add_forge_moderator.username,
actor_role="MODERATOR",
message_source=b"test with a message source",
)
rh.save()
api_client.force_login(add_forge_moderator)
url = reverse("api-1-add-forge-request-get", url_args={"id": resp.data["id"]})
resp = check_api_get_responses(api_client, url, status_code=200)
resp.data["history"] = [dict(history_item) for history_item in resp.data["history"]]
# Check that the authentified moderator can't urlhack non-existent message sources
assert resp.data["history"][0]["message_source_url"] is None
empty_message_url = reverse(
"forge-add-message-source", url_args={"id": resp.data["history"][0]["id"]}
)
check_http_get_response(api_client, empty_message_url, status_code=404)
# Check that the authentified moderator can't urlhack non-existent message sources
non_existent_message_url = reverse(
"forge-add-message-source", url_args={"id": 9001}
)
check_http_get_response(api_client, non_existent_message_url, status_code=404)
# Check that the authentified moderator can access the message source when the url is
# given
message_source_url = resp.data["history"][-1]["message_source_url"]
assert message_source_url is not None
message_source_resp = check_http_get_response(
api_client, message_source_url, status_code=200, content_type="text/email"
)
# Check that the message source shows up as an attachment
assert message_source_resp.content == rh.message_source
disposition = message_source_resp["Content-Disposition"]
assert disposition.startswith("attachment; filename=")
assert disposition.endswith('.eml"')
# Check that a regular user can't access message sources
api_client.force_login(regular_user)
check_http_get_response(api_client, message_source_url, status_code=302)
api_client.force_login(add_forge_moderator)
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_get_invalid(api_client):
url = reverse("api-1-add-forge-request-get", url_args={"id": 3})
check_api_get_responses(api_client, url, status_code=400)
diff --git a/swh/web/tests/add_forge_now/test_app.py b/swh/web/tests/add_forge_now/test_app.py
new file mode 100644
index 00000000..47cc92d8
--- /dev/null
+++ b/swh/web/tests/add_forge_now/test_app.py
@@ -0,0 +1,33 @@
+# Copyright (C) 2022 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU Affero General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+
+from django.urls import get_resolver
+
+from swh.web.add_forge_now.urls import urlpatterns
+from swh.web.common.utils import reverse
+from swh.web.tests.django_asserts import assert_not_contains
+from swh.web.tests.utils import check_html_get_response
+
+
+@pytest.mark.django_db
+def test_add_forge_now_deactivate(client, staff_user, django_settings):
+ """Check Add forge now feature is deactivated when the swh.web.add_forge_now django
+ application is not in installed apps."""
+
+ django_settings.SWH_DJANGO_APPS = [
+ app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.add_forge_now"
+ ]
+
+ url = reverse("swh-web-homepage")
+ client.force_login(staff_user)
+ resp = check_html_get_response(client, url, status_code=200)
+ assert_not_contains(resp, "swh-add-forge-now-item")
+ assert_not_contains(resp, "swh-add-forge-now-moderation-item")
+
+ add_forge_now_view_names = set(urlpattern.name for urlpattern in urlpatterns)
+ all_view_names = set(get_resolver().reverse_dict.keys())
+ assert add_forge_now_view_names & all_view_names == set()
diff --git a/swh/web/tests/add_forge_now/test_views.py b/swh/web/tests/add_forge_now/test_views.py
index 88750eae..a6a8eb23 100644
--- a/swh/web/tests/add_forge_now/test_views.py
+++ b/swh/web/tests/add_forge_now/test_views.py
@@ -1,221 +1,222 @@
# Copyright (C) 2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import pytest
from swh.web.common.utils import reverse
-from swh.web.tests.api.views.test_add_forge_now import create_add_forge_request
from swh.web.tests.utils import check_http_get_response
+from .test_api_views import create_add_forge_request
+
NB_FORGE_TYPE = 2
NB_FORGES_PER_TYPE = 20
def create_add_forge_requests(client, regular_user, regular_user2):
requests = []
for i in range(NB_FORGES_PER_TYPE):
request = {
"forge_type": "gitlab",
"forge_url": f"https://gitlab.example{i:02d}.org",
"forge_contact_email": f"admin@gitlab.example{i:02d}.org",
"forge_contact_name": f"gitlab.example{i:02d}.org admin",
"forge_contact_comment": "user marked as owner in forge members",
}
requests.append(
json.loads(
create_add_forge_request(
client,
regular_user,
data=request,
).content
)
)
request = {
"forge_type": "gitea",
"forge_url": f"https://gitea.example{i:02d}.org",
"forge_contact_email": f"admin@gitea.example{i:02d}.org",
"forge_contact_name": f"gitea.example{i:02d}.org admin",
"forge_contact_comment": "user marked as owner in forge members",
}
requests.append(
json.loads(
create_add_forge_request(
client,
regular_user2,
data=request,
).content
)
)
return requests
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_datatables_no_parameters(
client, regular_user, regular_user2
):
create_add_forge_requests(client, regular_user, regular_user2)
url = reverse("add-forge-request-list-datatables")
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
length = 10
assert data["draw"] == 0
assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == length
# default ordering is by descending id
assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1
assert "submitter_name" not in data["data"][0]
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_datatables(
client, regular_user, regular_user2, add_forge_moderator
):
create_add_forge_requests(client, regular_user, regular_user2)
length = 10
url = reverse(
"add-forge-request-list-datatables",
query_params={"draw": 1, "length": length, "start": 0},
)
client.force_login(regular_user)
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
assert data["draw"] == 1
assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == length
# default ordering is by descending id
assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1
assert "submitter_name" not in data["data"][0]
client.force_login(add_forge_moderator)
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
assert data["draw"] == 1
assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == length
# default ordering is by descending id
assert data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1
assert "submitter_name" in data["data"][0]
assert "last_moderator" in data["data"][0]
assert "last_modified_date" in data["data"][0]
return data
@pytest.mark.django_db(transaction=True, reset_sequences=True)
@pytest.mark.parametrize("order_field", ["forge_url", "last_modified_date"])
def test_add_forge_request_list_datatables_ordering(
client, add_forge_moderator, admin_user, order_field
):
requests = create_add_forge_requests(client, add_forge_moderator, admin_user)
requests_sorted = list(sorted(requests, key=lambda d: d[order_field]))
forge_urls_asc = [request[order_field] for request in requests_sorted]
forge_urls_desc = list(reversed(forge_urls_asc))
length = 10
client.force_login(admin_user)
for direction in ("asc", "desc"):
for i in range(4):
url = reverse(
"add-forge-request-list-datatables",
query_params={
"draw": 1,
"length": length,
"start": i * length,
"order[0][column]": 2,
"order[0][dir]": direction,
"columns[2][name]": order_field,
},
)
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
assert data["draw"] == 1
assert data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == length
page_forge_urls = [request[order_field] for request in data["data"]]
if direction == "asc":
expected_forge_urls = forge_urls_asc[i * length : (i + 1) * length]
else:
expected_forge_urls = forge_urls_desc[i * length : (i + 1) * length]
assert page_forge_urls == expected_forge_urls
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_datatables_search(client, regular_user, regular_user2):
create_add_forge_requests(client, regular_user, regular_user2)
url = reverse(
"add-forge-request-list-datatables",
query_params={
"draw": 1,
"length": NB_FORGES_PER_TYPE,
"start": 0,
"search[value]": "gitlab",
},
)
client.force_login(regular_user)
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
assert data["draw"] == 1
assert data["recordsFiltered"] == NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == NB_FORGES_PER_TYPE
page_forge_type = [request["forge_type"] for request in data["data"]]
assert page_forge_type == ["gitlab"] * NB_FORGES_PER_TYPE
@pytest.mark.django_db(transaction=True, reset_sequences=True)
def test_add_forge_request_list_datatables_user_requests(
client, regular_user, regular_user2
):
create_add_forge_requests(client, regular_user, regular_user2)
url = reverse(
"add-forge-request-list-datatables",
query_params={
"draw": 1,
"length": NB_FORGES_PER_TYPE * NB_FORGE_TYPE,
"start": 0,
"user_requests_only": 1,
},
)
client.force_login(regular_user2)
resp = check_http_get_response(client, url, status_code=200)
data = json.loads(resp.content)
assert data["draw"] == 1
assert data["recordsFiltered"] == NB_FORGES_PER_TYPE
assert data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE
assert len(data["data"]) == NB_FORGES_PER_TYPE
page_forge_type = [request["forge_type"] for request in data["data"]]
assert page_forge_type == ["gitea"] * NB_FORGES_PER_TYPE
diff --git a/swh/web/tests/test_config.py b/swh/web/tests/test_config.py
deleted file mode 100644
index 571b3be0..00000000
--- a/swh/web/tests/test_config.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright (C) 2022 The Software Heritage developers
-# See the AUTHORS file at the top-level directory of this distribution
-# License: GNU General Public License version 3, or any later version
-# See top-level LICENSE file for more information
-
-import pytest
-
-from swh.web.config import get_config, is_feature_enabled
-
-
-@pytest.mark.parametrize(
- "feature_name",
- ["inexistant-feature", "awesome-stuff"],
-)
-def test_is_feature_enabled(feature_name):
- config = get_config()
- # by default, feature non configured are considered disabled
- assert is_feature_enabled(feature_name) is False
-
- for enabled in [True, False]:
- # Let's configure the feature
- config["features"] = {feature_name: enabled}
- # and check its configuration is properly read
- assert is_feature_enabled(feature_name) is enabled
diff --git a/swh/web/urls.py b/swh/web/urls.py
index c0bf333c..72687d73 100644
--- a/swh/web/urls.py
+++ b/swh/web/urls.py
@@ -1,89 +1,86 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from importlib.util import find_spec
from django_js_reverse.views import urls_js
from django.conf import settings
from django.conf.urls import handler400, handler403, handler404, handler500, include
from django.contrib.auth.views import LogoutView
from django.contrib.staticfiles.views import serve
from django.shortcuts import render
from django.urls import re_path as url
from django.views.generic.base import RedirectView
from swh.web.browse.identifiers import swhid_browse
from swh.web.common.exc import (
swh_handle400,
swh_handle403,
swh_handle404,
swh_handle500,
)
from swh.web.common.utils import origin_visit_types
-from swh.web.config import get_config, is_feature_enabled
+from swh.web.config import get_config
swh_web_config = get_config()
favicon_view = RedirectView.as_view(
url="/static/img/icons/swh-logo-32x32.png", permanent=True
)
def _default_view(request):
return render(request, "homepage.html", {"visit_types": origin_visit_types()})
urlpatterns = [
url(r"^admin/", include("swh.web.admin.urls")),
url(r"^favicon\.ico/$", favicon_view),
url(r"^$", _default_view, name="swh-web-homepage"),
url(r"^jsreverse/$", urls_js, name="js_reverse"),
# keep legacy SWHID resolving URL with trailing slash for backward compatibility
url(
r"^(?P<swhid>(swh|SWH):[0-9]+:[A-Za-z]+:[0-9A-Fa-f]+.*)/$",
swhid_browse,
name="browse-swhid-legacy",
),
url(
r"^(?P<swhid>(swh|SWH):[0-9]+:[A-Za-z]+:[0-9A-Fa-f]+.*)$",
swhid_browse,
name="browse-swhid",
),
url(r"^", include("swh.web.misc.urls")),
url(r"^", include("swh.web.auth.views")),
url(r"^logout/$", LogoutView.as_view(template_name="logout.html"), name="logout"),
]
# Register URLs for each SWH Django application
for app in settings.SWH_DJANGO_APPS:
app_urls = app + ".urls"
try:
app_urls_spec = find_spec(app_urls)
if app_urls_spec is not None:
urlpatterns.append(url(r"^", include(app_urls)))
except ModuleNotFoundError:
assert False, f"Django application {app} not found !"
-if is_feature_enabled("add_forge_now"):
- urlpatterns += (url(r"^", include("swh.web.add_forge_now.views")),)
-
# allow to serve assets through django staticfiles
# even if settings.DEBUG is False
def insecure_serve(request, path, **kwargs):
return serve(request, path, insecure=True, **kwargs)
# enable to serve compressed assets through django development server
if swh_web_config["serve_assets"]:
static_pattern = r"^%s(?P<path>.*)/$" % settings.STATIC_URL[1:]
urlpatterns.append(url(static_pattern, insecure_serve))
handler400 = swh_handle400 # noqa
handler403 = swh_handle403 # noqa
handler404 = swh_handle404 # noqa
handler500 = swh_handle500 # noqa