diff --git a/swh/web/auth/apps.py b/swh/web/auth/apps.py index 45034773..d54212a4 100644 --- a/swh/web/auth/apps.py +++ b/swh/web/auth/apps.py @@ -1,11 +1,11 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.apps import AppConfig class AuthConfig(AppConfig): name = "swh.web.auth" - label = "swh.web.auth" + label = "swh_web_auth" diff --git a/swh/web/auth/migrations/0002_remove_stored_tokens.py b/swh/web/auth/migrations/0002_remove_stored_tokens.py index 5e1100a7..2e35dd45 100644 --- a/swh/web/auth/migrations/0002_remove_stored_tokens.py +++ b/swh/web/auth/migrations/0002_remove_stored_tokens.py @@ -1,22 +1,22 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import migrations from swh.web.auth.models import OIDCUserOfflineTokens def _remove_stored_encrypted_tokens(apps, schema_editor): OIDCUserOfflineTokens.objects.all().delete() class Migration(migrations.Migration): dependencies = [ - ("swh.web.auth", "0001_initial"), + ("swh_web_auth", "0001_initial"), ] operations = [migrations.RunPython(_remove_stored_encrypted_tokens)] diff --git a/swh/web/auth/models.py b/swh/web/auth/models.py index ee9a9fbe..e2b94a44 100644 --- a/swh/web/auth/models.py +++ b/swh/web/auth/models.py @@ -1,95 +1,95 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from datetime import datetime from typing import Optional, Set from django.contrib.auth.models import User from django.db import models class OIDCUser(User): """ Custom User proxy model for remote users storing OpenID Connect related data: profile containing authentication tokens. The model is also not saved to database as all users are already stored in the Keycloak one. """ # OIDC subject identifier sub: str = "" # OIDC tokens and session related data, only relevant when a user # authenticates from a web browser access_token: Optional[str] = None expires_at: Optional[datetime] = None id_token: Optional[str] = None refresh_token: Optional[str] = None refresh_expires_at: Optional[datetime] = None scope: Optional[str] = None session_state: Optional[str] = None # User permissions permissions: Set[str] class Meta: - app_label = "swh.web.auth" + app_label = "swh_web_auth" proxy = True def save(self, **kwargs): """ Override django.db.models.Model.save to avoid saving the remote users to web application database. """ pass def get_group_permissions(self, obj=None) -> Set[str]: """ Override django.contrib.auth.models.PermissionsMixin.get_group_permissions to get permissions from OIDC """ return self.get_all_permissions(obj) def get_all_permissions(self, obj=None) -> Set[str]: """ Override django.contrib.auth.models.PermissionsMixin.get_all_permissions to get permissions from OIDC """ return self.permissions def has_perm(self, perm, obj=None) -> bool: """ Override django.contrib.auth.models.PermissionsMixin.has_perm to check permission from OIDC """ if self.is_active and self.is_superuser: return True return perm in self.permissions def has_module_perms(self, app_label) -> bool: """ Override django.contrib.auth.models.PermissionsMixin.has_module_perms to check permissions from OIDC. """ if self.is_active and self.is_superuser: return True return any(perm.startswith(app_label) for perm in self.permissions) class OIDCUserOfflineTokens(models.Model): """ Model storing encrypted bearer tokens generated by users. """ user_id = models.CharField(max_length=50) creation_date = models.DateTimeField(auto_now_add=True) offline_token = models.BinaryField() class Meta: - app_label = "swh.web.auth" + app_label = "swh_web_auth" db_table = "oidc_user_offline_tokens" diff --git a/swh/web/common/apps.py b/swh/web/common/apps.py index f5727de8..f1e7582e 100644 --- a/swh/web/common/apps.py +++ b/swh/web/common/apps.py @@ -1,11 +1,11 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.apps import AppConfig class SwhWebCommonConfig(AppConfig): name = "swh.web.common" - label = "swh.web.common" + label = "swh_web_common" diff --git a/swh/web/common/migrations/0001_initial.py b/swh/web/common/migrations/0001_initial.py index ac4877e6..0061efeb 100644 --- a/swh/web/common/migrations/0001_initial.py +++ b/swh/web/common/migrations/0001_initial.py @@ -1,89 +1,89 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations, models _authorized_origins = [ "https://github.com/", "https://gitlab.com/", "https://bitbucket.org/", "https://git.code.sf.net/", "http://git.code.sf.net/", "https://hg.code.sf.net/", "http://hg.code.sf.net/", "https://svn.code.sf.net/", "http://svn.code.sf.net/", ] def _populate_save_authorized_origins(apps, schema_editor): - SaveAuthorizedOrigin = apps.get_model("swh.web.common", "SaveAuthorizedOrigin") + SaveAuthorizedOrigin = apps.get_model("swh_web_common", "SaveAuthorizedOrigin") for origin_url in _authorized_origins: SaveAuthorizedOrigin.objects.create(url=origin_url) class Migration(migrations.Migration): initial = True operations = [ migrations.CreateModel( name="SaveAuthorizedOrigin", fields=[ ( "id", models.AutoField( auto_created=True, primary_key=True, serialize=False, verbose_name="ID", ), ), ("url", models.CharField(max_length=200)), ], options={"db_table": "save_authorized_origin",}, ), migrations.CreateModel( name="SaveOriginRequest", fields=[ ("id", models.BigAutoField(primary_key=True, serialize=False)), ("request_date", models.DateTimeField(auto_now_add=True)), ("origin_type", models.CharField(max_length=200)), ("origin_url", models.CharField(max_length=200)), ( "status", models.TextField( choices=[ ("accepted", "accepted"), ("rejected", "rejected"), ("pending", "pending"), ], default="pending", ), ), ("loading_task_id", models.IntegerField(default=-1)), ], options={"db_table": "save_origin_request", "ordering": ["-id"],}, ), migrations.CreateModel( name="SaveUnauthorizedOrigin", fields=[ ( "id", models.AutoField( auto_created=True, primary_key=True, serialize=False, verbose_name="ID", ), ), ("url", models.CharField(max_length=200)), ], options={"db_table": "save_unauthorized_origin",}, ), migrations.RunPython(_populate_save_authorized_origins), ] diff --git a/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py b/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py index 73729ae4..b2792f2e 100644 --- a/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py +++ b/swh/web/common/migrations/0002_saveoriginrequest_visit_date.py @@ -1,23 +1,23 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0001_initial"), + ("swh_web_common", "0001_initial"), ] operations = [ migrations.AddField( model_name="saveoriginrequest", name="visit_date", field=models.DateTimeField(null=True), ), ] diff --git a/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py b/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py index ed2d2878..c539b675 100644 --- a/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py +++ b/swh/web/common/migrations/0003_saveoriginrequest_loading_task_status.py @@ -1,52 +1,52 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations, models from swh.web.config import scheduler def _remove_archived_tasks_with_no_saved_status(apps, schema_editor): """ Scheduler tasks are archived on a regular basis so their completion state could not be known anymore as previous to this migration, the loading task status was not stored in the database. So remove the rows associated to already archived tasks as the loading status can not be retrieved anymore. """ - SaveOriginRequest = apps.get_model("swh.web.common", "SaveOriginRequest") + SaveOriginRequest = apps.get_model("swh_web_common", "SaveOriginRequest") no_saved_status_tasks = [] for sor in SaveOriginRequest.objects.all(): tasks = scheduler().get_tasks([sor.loading_task_id]) if not tasks: no_saved_status_tasks.append(sor.loading_task_id) SaveOriginRequest.objects.filter(loading_task_id__in=no_saved_status_tasks).delete() class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0002_saveoriginrequest_visit_date"), + ("swh_web_common", "0002_saveoriginrequest_visit_date"), ] operations = [ migrations.AddField( model_name="saveoriginrequest", name="loading_task_status", field=models.TextField( choices=[ ("not created", "not created"), ("not yet scheduled", "not yet scheduled"), ("scheduled", "scheduled"), ("succeed", "succeed"), ("failed", "failed"), ], default="not created", ), ), migrations.RunPython(_remove_archived_tasks_with_no_saved_status), ] diff --git a/swh/web/common/migrations/0004_auto_20190204_1324.py b/swh/web/common/migrations/0004_auto_20190204_1324.py index 23513812..2021a315 100644 --- a/swh/web/common/migrations/0004_auto_20190204_1324.py +++ b/swh/web/common/migrations/0004_auto_20190204_1324.py @@ -1,33 +1,33 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0003_saveoriginrequest_loading_task_status"), + ("swh_web_common", "0003_saveoriginrequest_loading_task_status"), ] operations = [ migrations.AlterField( model_name="saveoriginrequest", name="loading_task_status", field=models.TextField( choices=[ ("not created", "not created"), ("not yet scheduled", "not yet scheduled"), ("scheduled", "scheduled"), ("succeed", "succeed"), ("failed", "failed"), ("running", "running"), ], default="not created", ), ), ] diff --git a/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py b/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py index e64f3093..748c3f53 100644 --- a/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py +++ b/swh/web/common/migrations/0005_remove_duplicated_authorized_origins.py @@ -1,25 +1,25 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations from swh.web.common.models import SaveAuthorizedOrigin def _remove_duplicated_urls_in_authorized_list(apps, schema_editor): sao = SaveAuthorizedOrigin.objects for url in sao.values_list("url", flat=True).distinct(): sao.filter(pk__in=sao.filter(url=url).values_list("id", flat=True)[1:]).delete() class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0004_auto_20190204_1324"), + ("swh_web_common", "0004_auto_20190204_1324"), ] operations = [migrations.RunPython(_remove_duplicated_urls_in_authorized_list)] diff --git a/swh/web/common/migrations/0006_rename_origin_type.py b/swh/web/common/migrations/0006_rename_origin_type.py index 7474fa7f..adbf4e6c 100644 --- a/swh/web/common/migrations/0006_rename_origin_type.py +++ b/swh/web/common/migrations/0006_rename_origin_type.py @@ -1,23 +1,23 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import unicode_literals from django.db import migrations class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0005_remove_duplicated_authorized_origins"), + ("swh_web_common", "0005_remove_duplicated_authorized_origins"), ] operations = [ migrations.RenameField( model_name="saveoriginrequest", old_name="origin_type", new_name="visit_type", ), ] diff --git a/swh/web/common/migrations/0007_save_request_task_status_fix_typo.py b/swh/web/common/migrations/0007_save_request_task_status_fix_typo.py index a2589c17..78f2c792 100644 --- a/swh/web/common/migrations/0007_save_request_task_status_fix_typo.py +++ b/swh/web/common/migrations/0007_save_request_task_status_fix_typo.py @@ -1,43 +1,43 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import migrations, models def _rename_request_status_from_succeed_to_succeeded(apps, schema_editor): """ Fix a typo in save request status value. """ - SaveOriginRequest = apps.get_model("swh.web.common", "SaveOriginRequest") + SaveOriginRequest = apps.get_model("swh_web_common", "SaveOriginRequest") for sor in SaveOriginRequest.objects.all(): if sor.loading_task_status == "succeed": sor.loading_task_status = "succeeded" sor.save() class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0006_rename_origin_type"), + ("swh_web_common", "0006_rename_origin_type"), ] operations = [ migrations.AlterField( model_name="saveoriginrequest", name="loading_task_status", field=models.TextField( choices=[ ("not created", "not created"), ("not yet scheduled", "not yet scheduled"), ("scheduled", "scheduled"), ("succeeded", "succeeded"), ("failed", "failed"), ("running", "running"), ], default="not created", ), ), migrations.RunPython(_rename_request_status_from_succeed_to_succeeded), ] diff --git a/swh/web/common/migrations/0008_save-code-now_indexes_20210106_1327.py b/swh/web/common/migrations/0008_save-code-now_indexes_20210106_1327.py index 70de0789..dd7afbb3 100644 --- a/swh/web/common/migrations/0008_save-code-now_indexes_20210106_1327.py +++ b/swh/web/common/migrations/0008_save-code-now_indexes_20210106_1327.py @@ -1,29 +1,29 @@ # Generated by Django 2.2.15 on 2021-01-06 13:27 # Adds indexes to the Save Code Now tables. from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ("swh.web.common", "0007_save_request_task_status_fix_typo"), + ("swh_web_common", "0007_save_request_task_status_fix_typo"), ] operations = [ migrations.AddIndex( model_name="saveauthorizedorigin", index=models.Index(fields=["url"], name="save_author_url_3e4e9d_idx"), ), migrations.AddIndex( model_name="saveoriginrequest", index=models.Index( fields=["origin_url", "status"], name="save_origin_origin__b46350_idx" ), ), migrations.AddIndex( model_name="saveunauthorizedorigin", index=models.Index(fields=["url"], name="save_unauth_url_c008fc_idx"), ), ] diff --git a/swh/web/common/models.py b/swh/web/common/models.py index 9c76f5ae..56ab34b2 100644 --- a/swh/web/common/models.py +++ b/swh/web/common/models.py @@ -1,102 +1,102 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.db import models class SaveAuthorizedOrigin(models.Model): """ Model table holding origin urls authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: - app_label = "swh.web.common" + app_label = "swh_web_common" db_table = "save_authorized_origin" indexes = [models.Index(fields=["url"])] def __str__(self): return self.url class SaveUnauthorizedOrigin(models.Model): """ Model table holding origin urls not authorized to be loaded into the archive. """ url = models.CharField(max_length=200, null=False) class Meta: - app_label = "swh.web.common" + app_label = "swh_web_common" db_table = "save_unauthorized_origin" indexes = [models.Index(fields=["url"])] def __str__(self): return self.url SAVE_REQUEST_ACCEPTED = "accepted" SAVE_REQUEST_REJECTED = "rejected" SAVE_REQUEST_PENDING = "pending" SAVE_REQUEST_STATUS = [ (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_ACCEPTED), (SAVE_REQUEST_REJECTED, SAVE_REQUEST_REJECTED), (SAVE_REQUEST_PENDING, SAVE_REQUEST_PENDING), ] SAVE_TASK_NOT_CREATED = "not created" SAVE_TASK_NOT_YET_SCHEDULED = "not yet scheduled" SAVE_TASK_SCHEDULED = "scheduled" SAVE_TASK_SUCCEEDED = "succeeded" SAVE_TASK_FAILED = "failed" SAVE_TASK_RUNNING = "running" SAVE_TASK_STATUS = [ (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_CREATED), (SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_NOT_YET_SCHEDULED), (SAVE_TASK_SCHEDULED, SAVE_TASK_SCHEDULED), (SAVE_TASK_SUCCEEDED, SAVE_TASK_SUCCEEDED), (SAVE_TASK_FAILED, SAVE_TASK_FAILED), (SAVE_TASK_RUNNING, SAVE_TASK_RUNNING), ] class SaveOriginRequest(models.Model): """ Model table holding all the save origin requests issued by users. """ id = models.BigAutoField(primary_key=True) request_date = models.DateTimeField(auto_now_add=True) visit_type = models.CharField(max_length=200, null=False) origin_url = models.CharField(max_length=200, null=False) status = models.TextField(choices=SAVE_REQUEST_STATUS, default=SAVE_REQUEST_PENDING) loading_task_id = models.IntegerField(default=-1) visit_date = models.DateTimeField(null=True) loading_task_status = models.TextField( choices=SAVE_TASK_STATUS, default=SAVE_TASK_NOT_CREATED ) class Meta: - app_label = "swh.web.common" + app_label = "swh_web_common" db_table = "save_origin_request" ordering = ["-id"] indexes = [models.Index(fields=["origin_url", "status"])] def __str__(self): return str( { "id": self.id, "request_date": self.request_date, "visit_type": self.visit_type, "origin_url": self.origin_url, "status": self.status, "loading_task_id": self.loading_task_id, "visit_date": self.visit_date, } )