diff --git a/swh/web/api/views/add_forge_now.py b/swh/web/api/views/add_forge_now.py index 13c89c82..b83d9572 100644 --- a/swh/web/api/views/add_forge_now.py +++ b/swh/web/api/views/add_forge_now.py @@ -1,352 +1,399 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json -from typing import Union +from typing import Any, Dict, Union from django.core.exceptions import ObjectDoesNotExist from django.core.paginator import Paginator from django.db import transaction +from django.db.models import Q from django.forms import CharField, ModelForm from django.http import HttpResponseBadRequest from django.http.request import HttpRequest from django.http.response import HttpResponse, HttpResponseForbidden from rest_framework import serializers from rest_framework.request import Request from rest_framework.response import Response from swh.web.add_forge_now.models import Request as AddForgeRequest from swh.web.add_forge_now.models import RequestActorRole as AddForgeNowRequestActorRole from swh.web.add_forge_now.models import RequestHistory as AddForgeNowRequestHistory from swh.web.add_forge_now.models import RequestStatus as AddForgeNowRequestStatus from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.common.exc import BadInputExc from swh.web.common.utils import reverse MODERATOR_ROLE = "swh.web.add_forge_now.moderator" def _block_while_testing(): """Replaced by tests to check concurrency behavior """ pass class AddForgeNowRequestForm(ModelForm): class Meta: model = AddForgeRequest fields = ( "forge_type", "forge_url", "forge_contact_email", "forge_contact_name", "forge_contact_comment", ) class AddForgeNowRequestHistoryForm(ModelForm): new_status = CharField(max_length=200, required=False,) class Meta: model = AddForgeNowRequestHistory fields = ("text", "new_status") class AddForgeNowRequestSerializer(serializers.ModelSerializer): class Meta: model = AddForgeRequest fields = "__all__" class AddForgeNowRequestPublicSerializer(serializers.ModelSerializer): """Serializes AddForgeRequest without private fields. """ class Meta: model = AddForgeRequest fields = ("id", "forge_url", "forge_type", "status", "submission_date") class AddForgeNowRequestHistorySerializer(serializers.ModelSerializer): class Meta: model = AddForgeNowRequestHistory exclude = ("request",) class AddForgeNowRequestHistoryPublicSerializer(serializers.ModelSerializer): class Meta: model = AddForgeNowRequestHistory fields = ("id", "date", "new_status", "actor_role") @api_route( r"/add-forge/request/create", "api-1-add-forge-request-create", methods=["POST"], ) @api_doc("/add-forge/request/create") @format_docstring() @transaction.atomic def api_add_forge_request_create(request: Union[HttpRequest, Request]) -> HttpResponse: """ .. http:post:: /api/1/add-forge/request/create/ Create a new request to add a forge to the list of those crawled regularly by Software Heritage. .. warning:: That endpoint is not publicly available and requires authentication in order to be able to request it. {common_headers} :[0-9]+)/update/", "api-1-add-forge-request-update", methods=["POST"], ) @api_doc("/add-forge/request/update", tags=["hidden"]) @format_docstring() @transaction.atomic def api_add_forge_request_update( request: Union[HttpRequest, Request], id: int ) -> HttpResponse: """ .. http:post:: /api/1/add-forge/request/update/ Update a request to add a forge to the list of those crawled regularly by Software Heritage. .. warning:: That endpoint is not publicly available and requires authentication in order to be able to request it. {common_headers} :[0-9]+)/get", "api-1-add-forge-request-get", methods=["GET"], ) @api_doc("/add-forge/request/get") @format_docstring() def api_add_forge_request_get(request: Request, id: int): """ .. http:get:: /api/1/add-forge/request/get/ Return all details about an add-forge request. {common_headers} :param int id: add-forge request identifier :statuscode 200: request details successfully returned :statuscode 400: request identifier does not exist """ try: add_forge_request = AddForgeRequest.objects.get(id=id) except ObjectDoesNotExist: raise BadInputExc("Request id does not exist") request_history = AddForgeNowRequestHistory.objects.filter( request=add_forge_request ).order_by("id") if request.user.is_authenticated and request.user.has_perm(MODERATOR_ROLE): data = AddForgeNowRequestSerializer(add_forge_request).data history = AddForgeNowRequestHistorySerializer(request_history, many=True).data else: data = AddForgeNowRequestPublicSerializer(add_forge_request).data history = AddForgeNowRequestHistoryPublicSerializer( request_history, many=True ).data return {"request": data, "history": history} diff --git a/swh/web/tests/api/views/test_add_forge_now.py b/swh/web/tests/api/views/test_add_forge_now.py index 59cbc9e4..d47ba548 100644 --- a/swh/web/tests/api/views/test_add_forge_now.py +++ b/swh/web/tests/api/views/test_add_forge_now.py @@ -1,489 +1,633 @@ # Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import threading import time from urllib.parse import urlencode import iso8601 import pytest from swh.web.add_forge_now.models import Request from swh.web.api.views.add_forge_now import MODERATOR_ROLE from swh.web.common.utils import reverse from swh.web.tests.utils import ( check_api_get_responses, check_api_post_response, check_http_post_response, create_django_permission, ) @pytest.mark.django_db def test_add_forge_request_create_anonymous_user(api_client): url = reverse("api-1-add-forge-request-create") check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db def test_add_forge_request_create_empty(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") resp = check_api_post_response(api_client, url, status_code=400) assert '"forge_type"' in resp.data["reason"] ADD_FORGE_DATA = { "forge_type": "gitlab", "forge_url": "https://gitlab.example.org", "forge_contact_email": "admin@gitlab.example.org", "forge_contact_name": "gitlab.example.org admin", "forge_contact_comment": "user marked as owner in forge members", } ADD_OTHER_FORGE_DATA = { "forge_type": "gitea", "forge_url": "https://gitea.example.org", "forge_contact_email": "admin@gitea.example.org", "forge_contact_name": "gitea.example.org admin", "forge_contact_comment": "user marked as owner in forge members", } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_success(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") date_before = datetime.datetime.now(tz=datetime.timezone.utc) resp = check_api_post_response( api_client, url, data=ADD_FORGE_DATA, status_code=201, ) date_after = datetime.datetime.now(tz=datetime.timezone.utc) assert resp.data == { **ADD_FORGE_DATA, "id": 1, "status": "PENDING", "submission_date": resp.data["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, } assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after request = Request.objects.all()[0] assert request.forge_url == ADD_FORGE_DATA["forge_url"] assert request.submitter_name == regular_user.username @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_success_form_encoded(client, regular_user): client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") date_before = datetime.datetime.now(tz=datetime.timezone.utc) resp = check_http_post_response( client, url, request_content_type="application/x-www-form-urlencoded", data=urlencode(ADD_FORGE_DATA), status_code=201, ) date_after = datetime.datetime.now(tz=datetime.timezone.utc) assert resp.data == { **ADD_FORGE_DATA, "id": 1, "status": "PENDING", "submission_date": resp.data["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, } assert date_before < iso8601.parse_date(resp.data["submission_date"]) < date_after request = Request.objects.all()[0] assert request.forge_url == ADD_FORGE_DATA["forge_url"] assert request.submitter_name == regular_user.username @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_create_duplicate(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") check_api_post_response( api_client, url, data=ADD_FORGE_DATA, status_code=201, ) check_api_post_response( api_client, url, data=ADD_FORGE_DATA, status_code=409, ) requests = Request.objects.all() assert len(requests) == 1 @pytest.fixture def moderator_user(regular_user2): regular_user2.user_permissions.add(create_django_permission(MODERATOR_ROLE)) return regular_user2 @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_anonymous_user(api_client): url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_regular_user(api_client, regular_user): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=403) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_non_existent(api_client, moderator_user): api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=400) def _create_add_forge_request(api_client, regular_user, data=ADD_FORGE_DATA): api_client.force_login(regular_user) url = reverse("api-1-add-forge-request-create") return check_api_post_response(api_client, url, data=data, status_code=201,) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_empty(api_client, regular_user, moderator_user): _create_add_forge_request(api_client, regular_user) api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, status_code=400) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_missing_field( api_client, regular_user, moderator_user ): _create_add_forge_request(api_client, regular_user) api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response(api_client, url, data={}, status_code=400) check_api_post_response( api_client, url, data={"new_status": "REJECTED"}, status_code=400 ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update(api_client, regular_user, moderator_user): _create_add_forge_request(api_client, regular_user) api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response( api_client, url, data={"text": "updating request"}, status_code=200 ) check_api_post_response( api_client, url, data={"new_status": "REJECTED", "text": "request rejected"}, status_code=200, ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_invalid_new_status( api_client, regular_user, moderator_user ): _create_add_forge_request(api_client, regular_user) api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) check_api_post_response( api_client, url, data={"new_status": "ACCEPTED", "text": "request accepted"}, status_code=400, ) @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_update_status_concurrent( api_client, regular_user, moderator_user, mocker ): _block_while_testing = mocker.patch( "swh.web.api.views.add_forge_now._block_while_testing" ) _block_while_testing.side_effect = lambda: time.sleep(1) _create_add_forge_request(api_client, regular_user) api_client.force_login(moderator_user) url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) worker_ended = False def worker(): nonlocal worker_ended check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) worker_ended = True # this thread will first modify the request status to WAITING_FOR_FEEDBACK thread = threading.Thread(target=worker) thread.start() # the other thread (slower) will attempt to modify the request status to REJECTED # but it will not be allowed as the first faster thread already modified it # and REJECTED state can not be reached from WAITING_FOR_FEEDBACK one time.sleep(0.5) check_api_post_response( api_client, url, data={"new_status": "REJECTED", "text": "request accepted"}, status_code=400, ) thread.join() assert worker_ended @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_anonymous(api_client, regular_user): url = reverse("api-1-add-forge-request-list") resp = check_api_get_responses(api_client, url, status_code=200) assert resp.data == [] _create_add_forge_request(api_client, regular_user) resp = check_api_get_responses(api_client, url, status_code=200) add_forge_request = { "forge_url": ADD_FORGE_DATA["forge_url"], "forge_type": ADD_FORGE_DATA["forge_type"], "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "id": 1, } assert resp.data == [add_forge_request] _create_add_forge_request(api_client, regular_user, data=ADD_OTHER_FORGE_DATA) resp = check_api_get_responses(api_client, url, status_code=200) other_forge_request = { "forge_url": ADD_OTHER_FORGE_DATA["forge_url"], "forge_type": ADD_OTHER_FORGE_DATA["forge_type"], "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "id": 2, } assert resp.data == [other_forge_request, add_forge_request] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_moderator(api_client, regular_user, moderator_user): url = reverse("api-1-add-forge-request-list") _create_add_forge_request(api_client, regular_user) _create_add_forge_request(api_client, regular_user, data=ADD_OTHER_FORGE_DATA) api_client.force_login(moderator_user) resp = check_api_get_responses(api_client, url, status_code=200) add_forge_request = { **ADD_FORGE_DATA, "status": "PENDING", "submission_date": resp.data[1]["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "id": 1, } other_forge_request = { **ADD_OTHER_FORGE_DATA, "status": "PENDING", "submission_date": resp.data[0]["submission_date"], "submitter_name": regular_user.username, "submitter_email": regular_user.email, "id": 2, } assert resp.data == [other_forge_request, add_forge_request] @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_pagination( api_client, regular_user, api_request_factory ): _create_add_forge_request(api_client, regular_user) _create_add_forge_request(api_client, regular_user, data=ADD_OTHER_FORGE_DATA) url = reverse("api-1-add-forge-request-list", query_params={"per_page": 1}) resp = check_api_get_responses(api_client, url, 200) assert len(resp.data) == 1 request = api_request_factory.get(url) next_url = reverse( "api-1-add-forge-request-list", query_params={"page": 2, "per_page": 1}, request=request, ) assert resp["Link"] == f'<{next_url}>; rel="next"' resp = check_api_get_responses(api_client, next_url, 200) assert len(resp.data) == 1 prev_url = reverse( "api-1-add-forge-request-list", query_params={"page": 1, "per_page": 1}, request=request, ) assert resp["Link"] == f'<{prev_url}>; rel="previous"' @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_list_submitter_filtering( api_client, regular_user, regular_user2 ): _create_add_forge_request(api_client, regular_user) _create_add_forge_request(api_client, regular_user2, data=ADD_OTHER_FORGE_DATA) api_client.force_login(regular_user) url = reverse( "api-1-add-forge-request-list", query_params={"user_requests_only": 1} ) resp = check_api_get_responses(api_client, url, status_code=200) assert len(resp.data) == 1 +NB_FORGE_TYPE = 2 +NB_FORGES_PER_TYPE = 20 + + +def _create_add_forge_requests(api_client, regular_user, regular_user2): + requests = [] + for i in range(NB_FORGES_PER_TYPE): + request = { + "forge_type": "gitlab", + "forge_url": f"https://gitlab.example{i:02d}.org", + "forge_contact_email": f"admin@gitlab.example{i:02d}.org", + "forge_contact_name": f"gitlab.example{i:02d}.org admin", + "forge_contact_comment": "user marked as owner in forge members", + } + _create_add_forge_request( + api_client, regular_user, data=request, + ) + requests.append(request) + + request = { + "forge_type": "gitea", + "forge_url": f"https://gitea.example{i:02d}.org", + "forge_contact_email": f"admin@gitea.example{i:02d}.org", + "forge_contact_name": f"gitea.example{i:02d}.org admin", + "forge_contact_comment": "user marked as owner in forge members", + } + _create_add_forge_request( + api_client, regular_user2, data=request, + ) + requests.append(request) + return requests + + +@pytest.mark.django_db(transaction=True, reset_sequences=True) +def test_add_forge_request_list_datatables( + api_client, regular_user, regular_user2, moderator_user +): + _create_add_forge_requests(api_client, regular_user, regular_user2) + + length = 10 + + url = reverse( + "api-1-add-forge-request-list", + query_params={"draw": 1, "length": length, "start": 0}, + ) + + api_client.force_login(regular_user) + resp = check_api_get_responses(api_client, url, status_code=200) + + assert resp.data["draw"] == 1 + assert resp.data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert resp.data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert len(resp.data["data"]) == length + # default ordering is by descending id + assert resp.data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert ( + resp.data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1 + ) + assert "submitter_name" not in resp.data["data"][0] + + api_client.force_login(moderator_user) + resp = check_api_get_responses(api_client, url, status_code=200) + + assert resp.data["draw"] == 1 + assert resp.data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert resp.data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert len(resp.data["data"]) == length + # default ordering is by descending id + assert resp.data["data"][0]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert ( + resp.data["data"][-1]["id"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE - length + 1 + ) + assert "submitter_name" in resp.data["data"][0] + + +@pytest.mark.django_db(transaction=True, reset_sequences=True) +def test_add_forge_request_list_datatables_ordering( + api_client, regular_user, regular_user2, moderator_user +): + requests = _create_add_forge_requests(api_client, regular_user, regular_user2) + requests_sorted = list(sorted(requests, key=lambda d: d["forge_url"])) + forge_urls_asc = [request["forge_url"] for request in requests_sorted] + forge_urls_desc = list(reversed(forge_urls_asc)) + + length = 10 + + for direction in ("asc", "desc"): + for i in range(4): + url = reverse( + "api-1-add-forge-request-list", + query_params={ + "draw": 1, + "length": length, + "start": i * length, + "order[0][column]": 2, + "order[0][dir]": direction, + "columns[2][name]": "forge_url", + }, + ) + + api_client.force_login(regular_user) + resp = check_api_get_responses(api_client, url, status_code=200) + + assert resp.data["draw"] == 1 + assert resp.data["recordsFiltered"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert resp.data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert len(resp.data["data"]) == length + + page_forge_urls = [request["forge_url"] for request in resp.data["data"]] + if direction == "asc": + expected_forge_urls = forge_urls_asc[i * length : (i + 1) * length] + else: + expected_forge_urls = forge_urls_desc[i * length : (i + 1) * length] + assert page_forge_urls == expected_forge_urls + + +@pytest.mark.django_db(transaction=True, reset_sequences=True) +def test_add_forge_request_list_datatables_search( + api_client, regular_user, regular_user2, moderator_user +): + _create_add_forge_requests(api_client, regular_user, regular_user2) + + url = reverse( + "api-1-add-forge-request-list", + query_params={ + "draw": 1, + "length": NB_FORGES_PER_TYPE, + "start": 0, + "search[value]": "gitlab", + }, + ) + + api_client.force_login(regular_user) + resp = check_api_get_responses(api_client, url, status_code=200) + + assert resp.data["draw"] == 1 + assert resp.data["recordsFiltered"] == NB_FORGES_PER_TYPE + assert resp.data["recordsTotal"] == NB_FORGE_TYPE * NB_FORGES_PER_TYPE + assert len(resp.data["data"]) == NB_FORGES_PER_TYPE + + page_forge_type = [request["forge_type"] for request in resp.data["data"]] + assert page_forge_type == ["gitlab"] * NB_FORGES_PER_TYPE + + @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get(api_client, regular_user, moderator_user): resp = _create_add_forge_request(api_client, regular_user) submission_date = resp.data["submission_date"] url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) api_client.force_login(moderator_user) check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) api_client.logout() url = reverse("api-1-add-forge-request-get", url_args={"id": 1}) resp = check_api_get_responses(api_client, url, status_code=200) assert resp.data == { "request": { "forge_url": ADD_FORGE_DATA["forge_url"], "forge_type": ADD_FORGE_DATA["forge_type"], "id": 1, "status": "WAITING_FOR_FEEDBACK", "submission_date": submission_date, }, "history": [ { "id": 1, "actor_role": "SUBMITTER", "date": resp.data["history"][0]["date"], "new_status": "PENDING", }, { "id": 2, "actor_role": "MODERATOR", "date": resp.data["history"][1]["date"], "new_status": "WAITING_FOR_FEEDBACK", }, ], } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get_moderator(api_client, regular_user, moderator_user): resp = _create_add_forge_request(api_client, regular_user) submission_date = resp.data["submission_date"] url = reverse("api-1-add-forge-request-update", url_args={"id": 1}) api_client.force_login(moderator_user) check_api_post_response( api_client, url, data={"new_status": "WAITING_FOR_FEEDBACK", "text": "waiting for message"}, status_code=200, ) url = reverse("api-1-add-forge-request-get", url_args={"id": 1}) resp = check_api_get_responses(api_client, url, status_code=200) assert resp.data == { "request": { **ADD_FORGE_DATA, "id": 1, "status": "WAITING_FOR_FEEDBACK", "submission_date": submission_date, "submitter_name": regular_user.username, "submitter_email": regular_user.email, }, "history": [ { "id": 1, "text": "", "actor": regular_user.username, "actor_role": "SUBMITTER", "date": resp.data["history"][0]["date"], "new_status": "PENDING", }, { "id": 2, "text": "waiting for message", "actor": moderator_user.username, "actor_role": "MODERATOR", "date": resp.data["history"][1]["date"], "new_status": "WAITING_FOR_FEEDBACK", }, ], } @pytest.mark.django_db(transaction=True, reset_sequences=True) def test_add_forge_request_get_invalid(api_client): url = reverse("api-1-add-forge-request-get", url_args={"id": 3}) check_api_get_responses(api_client, url, status_code=400)