diff --git a/swh/web/admin/mailmap.py b/swh/web/admin/mailmap.py deleted file mode 100644 --- a/swh/web/admin/mailmap.py +++ /dev/null @@ -1,16 +0,0 @@ -# Copyright (C) 2022 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from django.contrib.auth.decorators import permission_required -from django.shortcuts import render - -from swh.web.admin.adminurls import admin_route -from swh.web.auth.utils import MAILMAP_ADMIN_PERMISSION - - -@admin_route(r"mailmap/", view_name="admin-mailmap") -@permission_required(MAILMAP_ADMIN_PERMISSION) -def _admin_mailmap(request): - return render(request, "admin/mailmap.html") diff --git a/swh/web/admin/urls.py b/swh/web/admin/urls.py --- a/swh/web/admin/urls.py +++ b/swh/web/admin/urls.py @@ -9,7 +9,6 @@ from swh.web.admin.adminurls import AdminUrls import swh.web.admin.deposit # noqa -import swh.web.admin.mailmap # noqa import swh.web.admin.origin_save # noqa from swh.web.config import is_feature_enabled diff --git a/swh/web/auth/migrations/0007_mailmap_django_app.py b/swh/web/auth/migrations/0007_mailmap_django_app.py new file mode 100644 --- /dev/null +++ b/swh/web/auth/migrations/0007_mailmap_django_app.py @@ -0,0 +1,29 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ("swh_web_auth", "0006_fix_mailmap_admin_user_id"), + ] + + operations = [ + # as we simply move the mailmap feature to a dedicated django application, + # we do not want to remove the tables in database to not lose data + migrations.SeparateDatabaseAndState( + state_operations=[ + migrations.DeleteModel( + name="UserMailmap", + ), + migrations.DeleteModel( + name="UserMailmapEvent", + ), + ], + database_operations=[], + ), + ] diff --git a/swh/web/auth/models.py b/swh/web/auth/models.py --- a/swh/web/auth/models.py +++ b/swh/web/auth/models.py @@ -18,106 +18,3 @@ class Meta: app_label = "swh_web_auth" db_table = "oidc_user_offline_tokens" - - -class UserMailmapManager(models.Manager): - """A queryset manager which defers all :class:`models.DateTimeField` fields, to avoid - resetting them to an old value involuntarily.""" - - @classmethod - def deferred_fields(cls): - try: - return cls._deferred_fields - except AttributeError: - cls._deferred_fields = [ - field.name - for field in UserMailmap._meta.get_fields() - if isinstance(field, models.DateTimeField) and not field.auto_now - ] - return cls._deferred_fields - - def get_queryset(self): - return super().get_queryset().defer(*self.deferred_fields()) - - -class UserMailmap(models.Model): - """ - Model storing mailmap settings submitted by users. - """ - - user_id = models.CharField(max_length=50, null=True) - """Optional user id from Keycloak""" - - from_email = models.TextField(unique=True, null=False) - """Email address to find author in the archive""" - - from_email_verified = models.BooleanField(default=False) - """Indicates if the from email has been verified""" - - from_email_verification_request_date = models.DateTimeField(null=True) - """Last from email verification request date""" - - display_name = models.TextField(null=False) - """Display name to use for the author instead of the archived one""" - - display_name_activated = models.BooleanField(default=False) - """Indicates if the new display name should be used""" - - to_email = models.TextField(null=True) - """Optional new email to use in the display name instead of the archived one""" - - to_email_verified = models.BooleanField(default=False) - """Indicates if the to email has been verified""" - - to_email_verification_request_date = models.DateTimeField(null=True) - """Last to email verification request date""" - - mailmap_last_processing_date = models.DateTimeField(null=True) - """Last mailmap synchronisation date with swh-storage""" - - last_update_date = models.DateTimeField(auto_now=True) - """Last date that mailmap model was updated""" - - class Meta: - app_label = "swh_web_auth" - db_table = "user_mailmap" - - # Defer _date fields by default to avoid updating them by mistake - objects = UserMailmapManager() - - @property - def full_display_name(self) -> str: - if self.to_email is not None and self.to_email_verified: - return "%s <%s>" % (self.display_name, self.to_email) - else: - return self.display_name - - -class UserMailmapEvent(models.Model): - """ - Represents an update to a mailmap object - """ - - timestamp = models.DateTimeField(auto_now=True, null=False) - """Timestamp of the moment the event was submitted""" - - user_id = models.CharField(max_length=50, null=False) - """User id from Keycloak of the user who changed the mailmap. - (Not necessarily the one who the mail belongs to.)""" - - request_type = models.CharField(max_length=50, null=False) - """Either ``add`` or ``update``.""" - - request = models.TextField(null=False) - """JSON dump of the request received.""" - - successful = models.BooleanField(default=False, null=False) - """If False, then the request failed or crashed before completing, - and may or may not have altered the database's state.""" - - class Meta: - indexes = [ - models.Index(fields=["timestamp"]), - ] - app_label = "swh_web_auth" - db_table = "user_mailmap_event" diff --git a/swh/web/auth/views.py b/swh/web/auth/views.py --- a/swh/web/auth/views.py +++ b/swh/web/auth/views.py @@ -33,8 +33,6 @@ from swh.web.common.utils import reverse from swh.web.config import get_config -from .mailmap import urlpatterns as mailmap_urlpatterns - def oidc_generate_bearer_token(request: HttpRequest) -> HttpResponse: if not request.user.is_authenticated or not isinstance(request.user, OIDCUser): @@ -159,39 +157,35 @@ return render(request, "auth/profile.html") -urlpatterns = ( - auth_urlpatterns - + [ - url( - r"^oidc/generate-bearer-token/$", - oidc_generate_bearer_token, - name="oidc-generate-bearer-token", - ), - url( - r"^oidc/generate-bearer-token-complete/$", - oidc_generate_bearer_token_complete, - name="oidc-generate-bearer-token-complete", - ), - url( - r"^oidc/list-bearer-token/$", - oidc_list_bearer_tokens, - name="oidc-list-bearer-tokens", - ), - url( - r"^oidc/get-bearer-token/$", - oidc_get_bearer_token, - name="oidc-get-bearer-token", - ), - url( - r"^oidc/revoke-bearer-tokens/$", - oidc_revoke_bearer_tokens, - name="oidc-revoke-bearer-tokens", - ), - url( - r"^oidc/profile/$", - _oidc_profile_view, - name="oidc-profile", - ), - ] - + mailmap_urlpatterns -) +urlpatterns = auth_urlpatterns + [ + url( + r"^oidc/generate-bearer-token/$", + oidc_generate_bearer_token, + name="oidc-generate-bearer-token", + ), + url( + r"^oidc/generate-bearer-token-complete/$", + oidc_generate_bearer_token_complete, + name="oidc-generate-bearer-token-complete", + ), + url( + r"^oidc/list-bearer-token/$", + oidc_list_bearer_tokens, + name="oidc-list-bearer-tokens", + ), + url( + r"^oidc/get-bearer-token/$", + oidc_get_bearer_token, + name="oidc-get-bearer-token", + ), + url( + r"^oidc/revoke-bearer-tokens/$", + oidc_revoke_bearer_tokens, + name="oidc-revoke-bearer-tokens", + ), + url( + r"^oidc/profile/$", + _oidc_profile_view, + name="oidc-profile", + ), +] diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -160,6 +160,7 @@ [ "swh.web.inbound_email", "swh.web.add_forge_now", + "swh.web.mailmap", ], ), } diff --git a/swh/web/mailmap/__init__.py b/swh/web/mailmap/__init__.py new file mode 100644 --- /dev/null +++ b/swh/web/mailmap/__init__.py @@ -0,0 +1,6 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +default_app_config = "swh.web.mailmap.apps.MailmapConfig" diff --git a/swh/web/mailmap/apps.py b/swh/web/mailmap/apps.py new file mode 100644 --- /dev/null +++ b/swh/web/mailmap/apps.py @@ -0,0 +1,11 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.apps import AppConfig + + +class MailmapConfig(AppConfig): + name = "swh.web.mailmap" + label = "swh_web_mailmap" diff --git a/swh/web/auth/management/__init__.py b/swh/web/mailmap/management/__init__.py rename from swh/web/auth/management/__init__.py rename to swh/web/mailmap/management/__init__.py diff --git a/swh/web/auth/management/commands/__init__.py b/swh/web/mailmap/management/commands/__init__.py rename from swh/web/auth/management/commands/__init__.py rename to swh/web/mailmap/management/commands/__init__.py diff --git a/swh/web/auth/management/commands/sync_mailmaps.py b/swh/web/mailmap/management/commands/sync_mailmaps.py rename from swh/web/auth/management/commands/sync_mailmaps.py rename to swh/web/mailmap/management/commands/sync_mailmaps.py --- a/swh/web/auth/management/commands/sync_mailmaps.py +++ b/swh/web/mailmap/management/commands/sync_mailmaps.py @@ -13,7 +13,7 @@ from django.db.models.query import QuerySet from django.utils import timezone -from swh.web.auth.models import UserMailmap +from swh.web.mailmap.models import UserMailmap DISABLE_MAILMAPS_QUERY = """\ UPDATE person diff --git a/swh/web/mailmap/migrations/0001_initial.py b/swh/web/mailmap/migrations/0001_initial.py new file mode 100644 --- /dev/null +++ b/swh/web/mailmap/migrations/0001_initial.py @@ -0,0 +1,88 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [] # type: ignore + + operations = [ + migrations.SeparateDatabaseAndState( + # as we move the mailmap feature to a dedicated django application, + # no need to recreate database tables as they already exist + state_operations=[ + migrations.CreateModel( + name="UserMailmap", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("user_id", models.CharField(max_length=50, null=True)), + ("from_email", models.TextField(unique=True)), + ("from_email_verified", models.BooleanField(default=False)), + ( + "from_email_verification_request_date", + models.DateTimeField(null=True), + ), + ("display_name", models.TextField()), + ("display_name_activated", models.BooleanField(default=False)), + ("to_email", models.TextField(null=True)), + ("to_email_verified", models.BooleanField(default=False)), + ( + "to_email_verification_request_date", + models.DateTimeField(null=True), + ), + ( + "mailmap_last_processing_date", + models.DateTimeField(null=True), + ), + ("last_update_date", models.DateTimeField(auto_now=True)), + ], + options={ + "db_table": "user_mailmap", + }, + ), + migrations.CreateModel( + name="UserMailmapEvent", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("timestamp", models.DateTimeField(auto_now=True)), + ("user_id", models.CharField(max_length=50)), + ("request_type", models.CharField(max_length=50)), + ("request", models.TextField()), + ("successful", models.BooleanField(default=False)), + ], + options={ + "db_table": "user_mailmap_event", + }, + ), + migrations.AddIndex( + model_name="usermailmapevent", + index=models.Index( + fields=["timestamp"], name="user_mailma_timesta_1f7aef_idx" + ), + ), + ], + database_operations=[], + ), + ] diff --git a/swh/web/auth/management/__init__.py b/swh/web/mailmap/migrations/__init__.py copy from swh/web/auth/management/__init__.py copy to swh/web/mailmap/migrations/__init__.py diff --git a/swh/web/auth/models.py b/swh/web/mailmap/models.py copy from swh/web/auth/models.py copy to swh/web/mailmap/models.py --- a/swh/web/auth/models.py +++ b/swh/web/mailmap/models.py @@ -6,20 +6,6 @@ from django.db import models -class OIDCUserOfflineTokens(models.Model): - """ - Model storing encrypted bearer tokens generated by users. - """ - - user_id = models.CharField(max_length=50) - creation_date = models.DateTimeField(auto_now_add=True) - offline_token = models.BinaryField() - - class Meta: - app_label = "swh_web_auth" - db_table = "oidc_user_offline_tokens" - - class UserMailmapManager(models.Manager): """A queryset manager which defers all :class:`models.DateTimeField` fields, to avoid resetting them to an old value involuntarily.""" @@ -79,7 +65,7 @@ """Last date that mailmap model was updated""" class Meta: - app_label = "swh_web_auth" + app_label = "swh_web_mailmap" db_table = "user_mailmap" # Defer _date fields by default to avoid updating them by mistake @@ -119,5 +105,5 @@ indexes = [ models.Index(fields=["timestamp"]), ] - app_label = "swh_web_auth" + app_label = "swh_web_mailmap" db_table = "user_mailmap_event" diff --git a/swh/web/templates/admin/mailmap.html b/swh/web/mailmap/templates/admin/mailmap.html rename from swh/web/templates/admin/mailmap.html rename to swh/web/mailmap/templates/admin/mailmap.html diff --git a/swh/web/mailmap/urls.py b/swh/web/mailmap/urls.py new file mode 100644 --- /dev/null +++ b/swh/web/mailmap/urls.py @@ -0,0 +1,38 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.urls import re_path as url + +from swh.web.mailmap.views import ( + admin_mailmap, + profile_add_mailmap, + profile_list_mailmap, + profile_list_mailmap_datatables, + profile_update_mailmap, +) + +urlpatterns = [ + url( + r"^profile/mailmap/list/$", + profile_list_mailmap, + name="profile-mailmap-list", + ), + url( + r"^profile/mailmap/add/$", + profile_add_mailmap, + name="profile-mailmap-add", + ), + url( + r"^profile/mailmap/update/$", + profile_update_mailmap, + name="profile-mailmap-update", + ), + url( + r"^profile/mailmap/list/datatables/$", + profile_list_mailmap_datatables, + name="profile-mailmap-list-datatables", + ), + url(r"^admin/mailmap/$", admin_mailmap, name="admin-mailmap"), +] diff --git a/swh/web/auth/mailmap.py b/swh/web/mailmap/views.py rename from swh/web/auth/mailmap.py rename to swh/web/mailmap/views.py --- a/swh/web/auth/mailmap.py +++ b/swh/web/mailmap/views.py @@ -6,6 +6,7 @@ import json from typing import Any, Dict +from django.contrib.auth.decorators import permission_required from django.core.paginator import Paginator from django.db import IntegrityError from django.db.models import Q @@ -16,18 +17,18 @@ HttpResponseNotFound, JsonResponse, ) -from django.urls import re_path as url +from django.shortcuts import render from rest_framework import serializers from rest_framework.decorators import api_view from rest_framework.request import Request from rest_framework.response import Response -from swh.web.auth.models import UserMailmap, UserMailmapEvent from swh.web.auth.utils import ( MAILMAP_ADMIN_PERMISSION, MAILMAP_PERMISSION, any_permission_required, ) +from swh.web.mailmap.models import UserMailmap, UserMailmapEvent class UserMailmapSerializer(serializers.ModelSerializer): @@ -180,25 +181,6 @@ return JsonResponse(table_data) -urlpatterns = [ - url( - r"^profile/mailmap/list/$", - profile_list_mailmap, - name="profile-mailmap-list", - ), - url( - r"^profile/mailmap/add/$", - profile_add_mailmap, - name="profile-mailmap-add", - ), - url( - r"^profile/mailmap/update/$", - profile_update_mailmap, - name="profile-mailmap-update", - ), - url( - r"^profile/mailmap/list/datatables/$", - profile_list_mailmap_datatables, - name="profile-mailmap-list-datatables", - ), -] +@permission_required(MAILMAP_ADMIN_PERMISSION) +def admin_mailmap(request): + return render(request, "admin/mailmap.html") diff --git a/swh/web/templates/layout.html b/swh/web/templates/layout.html --- a/swh/web/templates/layout.html +++ b/swh/web/templates/layout.html @@ -249,7 +249,7 @@ {% endif %} - {% if MAILMAP_ADMIN_PERMISSION in user.get_all_permissions %} + {% if "swh.web.mailmap" in SWH_DJANGO_APPS and MAILMAP_ADMIN_PERMISSION in user.get_all_permissions %}
  • diff --git a/swh/web/tests/auth/test_migrations.py b/swh/web/tests/auth/test_migrations.py --- a/swh/web/tests/auth/test_migrations.py +++ b/swh/web/tests/auth/test_migrations.py @@ -5,10 +5,13 @@ from datetime import datetime +import pytest + APP_NAME = "swh_web_auth" MIGRATION_0005 = "0005_usermailmapevent" MIGRATION_0006 = "0006_fix_mailmap_admin_user_id" +MIGRATION_0007 = "0007_mailmap_django_app" def test_fix_mailmap_admin_user_id(migrator): @@ -36,3 +39,16 @@ assert UserMailmap.objects.filter(user_id=user_id).count() == 0 assert UserMailmap.objects.filter(user_id=None).count() == 1 + + +def test_mailmap_django_app(migrator): + state = migrator.apply_tested_migration((APP_NAME, MIGRATION_0006)) + UserMailmap = state.apps.get_model(APP_NAME, "UserMailmap") + assert UserMailmap + + # UserMailmap model moved to swh_web_mailmap django application + state = migrator.apply_tested_migration((APP_NAME, MIGRATION_0007)) + with pytest.raises( + LookupError, match="App 'swh_web_auth' doesn't have a 'UserMailmap' model." + ): + state.apps.get_model(APP_NAME, "UserMailmap") diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -16,7 +16,8 @@ from typing import Any, Dict, List, Optional from _pytest.python import Function -from hypothesis import HealthCheck, settings +from hypothesis import HealthCheck +from hypothesis import settings as hypothesis_settings import pytest from pytest_django.fixtures import SettingsWrapper @@ -64,7 +65,7 @@ fossology_missing = shutil.which("nomossa") is None # Register some hypothesis profiles -settings.register_profile("default", settings()) +hypothesis_settings.register_profile("default", hypothesis_settings()) # we use getattr here to keep mypy happy regardless hypothesis version function_scoped_fixture_check = ( @@ -79,17 +80,17 @@ ] + function_scoped_fixture_check -settings.register_profile( +hypothesis_settings.register_profile( "swh-web", - settings( + hypothesis_settings( deadline=None, suppress_health_check=suppress_health_check, ), ) -settings.register_profile( +hypothesis_settings.register_profile( "swh-web-fast", - settings( + hypothesis_settings( deadline=None, max_examples=5, suppress_health_check=suppress_health_check, @@ -105,7 +106,7 @@ # Use fast hypothesis profile by default if none has been # explicitly specified in pytest option if config.getoption("--hypothesis-profile") is None: - settings.load_profile("swh-web-fast") + hypothesis_settings.load_profile("swh-web-fast") # Small hack in order to be able to run the unit tests # without static assets generated by webpack. diff --git a/swh/web/auth/management/__init__.py b/swh/web/tests/mailmap/__init__.py rename from swh/web/auth/management/__init__.py rename to swh/web/tests/mailmap/__init__.py diff --git a/swh/web/tests/mailmap/test_app.py b/swh/web/tests/mailmap/test_app.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/mailmap/test_app.py @@ -0,0 +1,32 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import pytest + +from django.urls import get_resolver + +from swh.web.common.utils import reverse +from swh.web.mailmap.urls import urlpatterns +from swh.web.tests.django_asserts import assert_not_contains +from swh.web.tests.utils import check_html_get_response + + +@pytest.mark.django_db +def test_mailmap_deactivate(client, mailmap_admin, django_settings): + """Check mailmap feature is deactivated when the swh.web.mailmap django + application is not in installed apps.""" + + django_settings.SWH_DJANGO_APPS = [ + app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.mailmap" + ] + + url = reverse("swh-web-homepage") + client.force_login(mailmap_admin) + resp = check_html_get_response(client, url, status_code=200) + assert_not_contains(resp, "swh-mailmap-admin-item") + + mailmap_view_names = set(urlpattern.name for urlpattern in urlpatterns) + all_view_names = set(get_resolver().reverse_dict.keys()) + assert mailmap_view_names & all_view_names == set() diff --git a/swh/web/tests/auth/test_mailmap.py b/swh/web/tests/mailmap/test_mailmap.py rename from swh/web/tests/auth/test_mailmap.py rename to swh/web/tests/mailmap/test_mailmap.py --- a/swh/web/tests/auth/test_mailmap.py +++ b/swh/web/tests/mailmap/test_mailmap.py @@ -15,8 +15,8 @@ from django.db import transaction from swh.model.model import Person -from swh.web.auth.models import UserMailmap, UserMailmapEvent from swh.web.common.utils import reverse +from swh.web.mailmap.models import UserMailmap, UserMailmapEvent from swh.web.tests.utils import ( check_api_post_response, check_http_get_response, diff --git a/swh/web/tests/mailmap/test_migrations.py b/swh/web/tests/mailmap/test_migrations.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/mailmap/test_migrations.py @@ -0,0 +1,15 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +APP_NAME = "swh_web_mailmap" + +MIGRATION_0001 = "0001_initial" + + +def test_mailmap_django_app(migrator): + state = migrator.apply_tested_migration((APP_NAME, MIGRATION_0001)) + UserMailmap = state.apps.get_model(APP_NAME, "UserMailmap") + assert UserMailmap