diff --git a/docs/developers-info.rst b/docs/developers-info.rst --- a/docs/developers-info.rst +++ b/docs/developers-info.rst @@ -60,21 +60,21 @@ Common utilities """""""""""""""" - * :mod:`swh.web.common.converters`: conversion module used to transform raw data - to serializable ones. It is used by :mod:`swh.web.common.archive`: to convert data + * :mod:`swh.web.utils.converters`: conversion module used to transform raw data + to serializable ones. It is used by :mod:`swh.web.utils.archive`: to convert data before transmitting then to Django views. - * :mod:`swh.web.common.exc`: module defining exceptions used in the web applications. - * :mod:`swh.web.common.highlightjs`: utility module to ease the use of the highlightjs_ + * :mod:`swh.web.utils.exc`: module defining exceptions used in the web applications. + * :mod:`swh.web.utils.highlightjs`: utility module to ease the use of the highlightjs_ library in produced Django views. - * :mod:`swh.web.common.query`: Utilities to parse data from HTTP endpoints. It is used - by :mod:`swh.web.common.archive`. - * :mod:`swh.web.common.archive`: Orchestration layer used by views module + * :mod:`swh.web.utils.query`: Utilities to parse data from HTTP endpoints. It is used + by :mod:`swh.web.utils.archive`. + * :mod:`swh.web.utils.archive`: Orchestration layer used by views module in charge of communication with :mod:`swh.storage` to retrieve information and perform conversion for the upper layer. - * :mod:`swh.web.common.swh_templatetags`: Custom Django template tags library for swh. - * :mod:`swh.web.common.urlsindex`: Utilities to help the registering of endpoints + * :mod:`swh.web.utils.swh_templatetags`: Custom Django template tags library for swh. + * :mod:`swh.web.utils.urlsindex`: Utilities to help the registering of endpoints for the web applications - * :mod:`swh.web.common.utils`: Utility functions used in the web applications implementation + * :mod:`swh.web.utils`: Utility functions used in the web applications implementation swh-web API application diff --git a/swh/web/add_forge_now/api_views.py b/swh/web/add_forge_now/api_views.py --- a/swh/web/add_forge_now/api_views.py +++ b/swh/web/add_forge_now/api_views.py @@ -24,8 +24,8 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.auth.utils import is_add_forge_now_moderator -from swh.web.common.exc import BadInputExc -from swh.web.common.utils import reverse +from swh.web.utils import reverse +from swh.web.utils.exc import BadInputExc def _block_while_testing(): diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -19,7 +19,7 @@ from swh.web.api.apiresponse import make_api_response from swh.web.api.apiurls import APIUrls -from swh.web.common.utils import parse_rst +from swh.web.utils import parse_rst class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py --- a/swh/web/api/apiresponse.py +++ b/swh/web/api/apiresponse.py @@ -19,15 +19,15 @@ from swh.storage.exc import StorageAPIError, StorageDBError from swh.web.api import utils -from swh.web.common.exc import ( +from swh.web.config import get_config +from swh.web.utils import gen_path_info, shorten_path +from swh.web.utils.exc import ( BadInputExc, ForbiddenExc, LargePayloadExc, NotFoundExc, sentry_capture_exception, ) -from swh.web.common.utils import gen_path_info, shorten_path -from swh.web.config import get_config logger = logging.getLogger("django") diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -11,7 +11,7 @@ from swh.web.api import throttling from swh.web.api.apiresponse import make_api_response -from swh.web.common.urlsindex import UrlsIndex +from swh.web.utils.urlsindex import UrlsIndex class APIUrls(UrlsIndex): diff --git a/swh/web/api/throttling.py b/swh/web/api/throttling.py --- a/swh/web/api/throttling.py +++ b/swh/web/api/throttling.py @@ -11,8 +11,8 @@ from rest_framework.throttling import ScopedRateThrottle from swh.web.auth.utils import API_RAW_OBJECT_PERMISSION, API_SAVE_ORIGIN_PERMISSION -from swh.web.common.exc import sentry_capture_exception from swh.web.config import get_config +from swh.web.utils.exc import sentry_capture_exception APIView = TypeVar("APIView", bound="rest_framework.views.APIView") Request = rest_framework.request.Request diff --git a/swh/web/api/utils.py b/swh/web/api/utils.py --- a/swh/web/api/utils.py +++ b/swh/web/api/utils.py @@ -8,9 +8,9 @@ from django.http import HttpRequest from swh.model.model import Origin -from swh.web.common.query import parse_hash -from swh.web.common.typing import OriginInfo -from swh.web.common.utils import resolve_branch_alias, reverse +from swh.web.utils import resolve_branch_alias, reverse +from swh.web.utils.query import parse_hash +from swh.web.utils.typing import OriginInfo def filter_field_keys(data, field_keys): diff --git a/swh/web/api/views/content.py b/swh/web/api/views/content.py --- a/swh/web/api/views/content.py +++ b/swh/web/api/views/content.py @@ -14,8 +14,8 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -from swh.web.common import archive -from swh.web.common.exc import NotFoundExc +from swh.web.utils import archive +from swh.web.utils.exc import NotFoundExc @api_route( diff --git a/swh/web/api/views/directory.py b/swh/web/api/views/directory.py --- a/swh/web/api/views/directory.py +++ b/swh/web/api/views/directory.py @@ -11,7 +11,7 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -from swh.web.common import archive +from swh.web.utils import archive @api_route( diff --git a/swh/web/api/views/graph.py b/swh/web/api/views/graph.py --- a/swh/web/api/views/graph.py +++ b/swh/web/api/views/graph.py @@ -23,8 +23,8 @@ from swh.web.api.apidoc import api_doc from swh.web.api.apiurls import api_route from swh.web.api.renderers import PlainTextRenderer -from swh.web.common import archive from swh.web.config import SWH_WEB_INTERNAL_SERVER_NAME, get_config +from swh.web.utils import archive API_GRAPH_PERM = "swh.web.api.graph" diff --git a/swh/web/api/views/identifiers.py b/swh/web/api/views/identifiers.py --- a/swh/web/api/views/identifiers.py +++ b/swh/web/api/views/identifiers.py @@ -11,9 +11,9 @@ from swh.model.swhids import ObjectType from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route -from swh.web.common import archive -from swh.web.common.exc import LargePayloadExc -from swh.web.common.identifiers import get_swhid, group_swhids, resolve_swhid +from swh.web.utils import archive +from swh.web.utils.exc import LargePayloadExc +from swh.web.utils.identifiers import get_swhid, group_swhids, resolve_swhid @api_route(r"/resolve/(?P.+)/", "api-1-resolve-swhid") diff --git a/swh/web/api/views/metadata.py b/swh/web/api/views/metadata.py --- a/swh/web/api/views/metadata.py +++ b/swh/web/api/views/metadata.py @@ -17,9 +17,8 @@ from swh.model.model import MetadataAuthority, MetadataAuthorityType, Origin from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route -from swh.web.common import archive, converters -from swh.web.common.exc import BadInputExc, NotFoundExc -from swh.web.common.utils import SWHID_RE, reverse +from swh.web.utils import SWHID_RE, archive, converters, reverse +from swh.web.utils.exc import BadInputExc, NotFoundExc @api_route( diff --git a/swh/web/api/views/origin.py b/swh/web/api/views/origin.py --- a/swh/web/api/views/origin.py +++ b/swh/web/api/views/origin.py @@ -18,11 +18,10 @@ enrich_origin_visit, ) from swh.web.api.views.utils import api_lookup -from swh.web.common import archive -from swh.web.common.exc import BadInputExc -from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.typing import OriginInfo -from swh.web.common.utils import origin_visit_types, reverse +from swh.web.utils import archive, origin_visit_types, reverse +from swh.web.utils.exc import BadInputExc +from swh.web.utils.origin_visits import get_origin_visits +from swh.web.utils.typing import OriginInfo DOC_RETURN_ORIGIN = """ :>json string origin_visits_url: link to in order to get information diff --git a/swh/web/api/views/raw.py b/swh/web/api/views/raw.py --- a/swh/web/api/views/raw.py +++ b/swh/web/api/views/raw.py @@ -22,9 +22,8 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.auth.utils import API_RAW_OBJECT_PERMISSION -from swh.web.common import archive -from swh.web.common.exc import NotFoundExc -from swh.web.common.utils import SWHID_RE +from swh.web.utils import SWHID_RE, archive +from swh.web.utils.exc import NotFoundExc @api_route( diff --git a/swh/web/api/views/release.py b/swh/web/api/views/release.py --- a/swh/web/api/views/release.py +++ b/swh/web/api/views/release.py @@ -9,7 +9,7 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -from swh.web.common import archive +from swh.web.utils import archive @api_route( diff --git a/swh/web/api/views/revision.py b/swh/web/api/views/revision.py --- a/swh/web/api/views/revision.py +++ b/swh/web/api/views/revision.py @@ -12,7 +12,7 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -from swh.web.common import archive +from swh.web.utils import archive DOC_RETURN_REVISION = """ :>json object author: information about the author of the revision diff --git a/swh/web/api/views/snapshot.py b/swh/web/api/views/snapshot.py --- a/swh/web/api/views/snapshot.py +++ b/swh/web/api/views/snapshot.py @@ -9,9 +9,8 @@ from swh.web.api.apiurls import api_route from swh.web.api.utils import enrich_snapshot from swh.web.api.views.utils import api_lookup -from swh.web.common import archive -from swh.web.common.utils import reverse from swh.web.config import get_config +from swh.web.utils import archive, reverse @api_route( diff --git a/swh/web/api/views/stat.py b/swh/web/api/views/stat.py --- a/swh/web/api/views/stat.py +++ b/swh/web/api/views/stat.py @@ -7,7 +7,7 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route -from swh.web.common import archive +from swh.web.utils import archive @api_route(r"/stat/counters/", "api-1-stat-counters") diff --git a/swh/web/api/views/utils.py b/swh/web/api/views/utils.py --- a/swh/web/api/views/utils.py +++ b/swh/web/api/views/utils.py @@ -12,7 +12,7 @@ from rest_framework.response import Response from swh.web.api.apiurls import APIUrls, api_route -from swh.web.common.exc import NotFoundExc +from swh.web.utils.exc import NotFoundExc EnrichFunction = Callable[[Dict[str, str], Optional[HttpRequest]], Dict[str, str]] diff --git a/swh/web/api/views/vault.py b/swh/web/api/views/vault.py --- a/swh/web/api/views/vault.py +++ b/swh/web/api/views/vault.py @@ -14,9 +14,8 @@ from swh.web.api.apidoc import api_doc, format_docstring from swh.web.api.apiurls import api_route from swh.web.api.views.utils import api_lookup -from swh.web.common import archive, query -from swh.web.common.exc import BadInputExc -from swh.web.common.utils import SWHID_RE, reverse +from swh.web.utils import SWHID_RE, archive, query, reverse +from swh.web.utils.exc import BadInputExc ###################################################### # Common diff --git a/swh/web/auth/utils.py b/swh/web/auth/utils.py --- a/swh/web/auth/utils.py +++ b/swh/web/auth/utils.py @@ -14,9 +14,7 @@ from django.contrib.auth.decorators import user_passes_test from django.http.request import HttpRequest -from swh.web.common.exc import ForbiddenExc - -OIDC_SWH_WEB_CLIENT_ID = "swh-web" +from swh.web.utils.exc import ForbiddenExc SWH_AMBASSADOR_PERMISSION = "swh.ambassador" API_SAVE_ORIGIN_PERMISSION = "swh.web.api.save_origin" diff --git a/swh/web/auth/views.py b/swh/web/auth/views.py --- a/swh/web/auth/views.py +++ b/swh/web/auth/views.py @@ -27,9 +27,9 @@ from swh.auth.keycloak import KeycloakError, keycloak_error_message from swh.web.auth.models import OIDCUserOfflineTokens from swh.web.auth.utils import decrypt_data, encrypt_data -from swh.web.common.exc import ForbiddenExc -from swh.web.common.utils import reverse from swh.web.config import get_config +from swh.web.utils import reverse +from swh.web.utils.exc import ForbiddenExc def oidc_generate_bearer_token(request: HttpRequest) -> HttpResponse: diff --git a/swh/web/browse/browseurls.py b/swh/web/browse/browseurls.py --- a/swh/web/browse/browseurls.py +++ b/swh/web/browse/browseurls.py @@ -5,7 +5,7 @@ from typing import List, Optional -from swh.web.common.urlsindex import UrlsIndex +from swh.web.utils.urlsindex import UrlsIndex class BrowseUrls(UrlsIndex): diff --git a/swh/web/browse/identifiers.py b/swh/web/browse/identifiers.py --- a/swh/web/browse/identifiers.py +++ b/swh/web/browse/identifiers.py @@ -6,7 +6,7 @@ from django.http import HttpRequest, HttpResponse from django.shortcuts import redirect -from swh.web.common.identifiers import resolve_swhid +from swh.web.utils.identifiers import resolve_swhid def swhid_browse(request: HttpRequest, swhid: str) -> HttpResponse: diff --git a/swh/web/browse/snapshot_context.py b/swh/web/browse/snapshot_context.py --- a/swh/web/browse/snapshot_context.py +++ b/swh/web/browse/snapshot_context.py @@ -25,11 +25,19 @@ get_directory_entries, get_readme_to_display, ) -from swh.web.common import archive -from swh.web.common.exc import BadInputExc, NotFoundExc, http_status_code_message -from swh.web.common.identifiers import get_swhids_info -from swh.web.common.origin_visits import get_origin_visit -from swh.web.common.typing import ( +from swh.web.config import get_config +from swh.web.utils import ( + archive, + django_cache, + format_utc_iso_date, + gen_path_info, + reverse, + swh_object_icons, +) +from swh.web.utils.exc import BadInputExc, NotFoundExc, http_status_code_message +from swh.web.utils.identifiers import get_swhids_info +from swh.web.utils.origin_visits import get_origin_visit +from swh.web.utils.typing import ( DirectoryMetadata, OriginInfo, SnapshotBranchInfo, @@ -37,14 +45,6 @@ SnapshotReleaseInfo, SWHObjectInfo, ) -from swh.web.common.utils import ( - django_cache, - format_utc_iso_date, - gen_path_info, - reverse, - swh_object_icons, -) -from swh.web.config import get_config _empty_snapshot_id = Snapshot(branches={}).id.hex() @@ -181,7 +181,7 @@ Args: snapshot: A dict describing a snapshot as returned for instance by - :func:`swh.web.common.archive.lookup_snapshot` + :func:`swh.web.utils.archive.lookup_snapshot` Returns: A tuple whose first member is the sorted list of branches @@ -416,7 +416,7 @@ A dict filled with snapshot context information. Raises: - swh.web.common.exc.NotFoundExc: if no snapshot is found for the visit + swh.web.utils.exc.NotFoundExc: if no snapshot is found for the visit of an origin. """ assert origin_url is not None or snapshot_id is not None diff --git a/swh/web/browse/urls.py b/swh/web/browse/urls.py --- a/swh/web/browse/urls.py +++ b/swh/web/browse/urls.py @@ -15,7 +15,7 @@ import swh.web.browse.views.release # noqa import swh.web.browse.views.revision # noqa import swh.web.browse.views.snapshot # noqa -from swh.web.common.utils import origin_visit_types, reverse +from swh.web.utils import origin_visit_types, reverse def _browse_help_view(request: HttpRequest) -> HttpResponse: diff --git a/swh/web/browse/utils.py b/swh/web/browse/utils.py --- a/swh/web/browse/utils.py +++ b/swh/web/browse/utils.py @@ -14,17 +14,18 @@ from django.utils.html import escape from django.utils.safestring import mark_safe -from swh.web.common import archive, highlightjs -from swh.web.common.exc import NotFoundExc, sentry_capture_exception -from swh.web.common.typing import SnapshotContext -from swh.web.common.utils import ( +from swh.web.config import get_config +from swh.web.utils import ( + archive, browsers_supported_image_mimes, django_cache, format_utc_iso_date, + highlightjs, reverse, rst_to_html, ) -from swh.web.config import get_config +from swh.web.utils.exc import NotFoundExc, sentry_capture_exception +from swh.web.utils.typing import SnapshotContext @django_cache() diff --git a/swh/web/browse/views/content.py b/swh/web/browse/views/content.py --- a/swh/web/browse/views/content.py +++ b/swh/web/browse/views/content.py @@ -20,16 +20,22 @@ prepare_content_for_display, request_content, ) -from swh.web.common import archive, highlightjs, query -from swh.web.common.exc import ( +from swh.web.utils import ( + archive, + gen_path_info, + highlightjs, + query, + reverse, + swh_object_icons, +) +from swh.web.utils.exc import ( BadInputExc, NotFoundExc, http_status_code_message, sentry_capture_exception, ) -from swh.web.common.identifiers import get_swhids_info -from swh.web.common.typing import ContentMetadata, SWHObjectInfo -from swh.web.common.utils import gen_path_info, reverse, swh_object_icons +from swh.web.utils.identifiers import get_swhids_info +from swh.web.utils.typing import ContentMetadata, SWHObjectInfo @browse_route( diff --git a/swh/web/browse/views/directory.py b/swh/web/browse/views/directory.py --- a/swh/web/browse/views/directory.py +++ b/swh/web/browse/views/directory.py @@ -13,15 +13,14 @@ from swh.web.browse.browseurls import browse_route from swh.web.browse.snapshot_context import get_snapshot_context from swh.web.browse.utils import gen_link, get_directory_entries, get_readme_to_display -from swh.web.common import archive -from swh.web.common.exc import ( +from swh.web.utils import archive, gen_path_info, reverse, swh_object_icons +from swh.web.utils.exc import ( NotFoundExc, http_status_code_message, sentry_capture_exception, ) -from swh.web.common.identifiers import get_swhids_info -from swh.web.common.typing import DirectoryMetadata, SWHObjectInfo -from swh.web.common.utils import gen_path_info, reverse, swh_object_icons +from swh.web.utils.identifiers import get_swhids_info +from swh.web.utils.typing import DirectoryMetadata, SWHObjectInfo def _directory_browse( diff --git a/swh/web/browse/views/origin.py b/swh/web/browse/views/origin.py --- a/swh/web/browse/views/origin.py +++ b/swh/web/browse/views/origin.py @@ -13,15 +13,15 @@ browse_snapshot_directory, get_snapshot_context, ) -from swh.web.common import archive -from swh.web.common.exc import BadInputExc -from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.utils import ( +from swh.web.utils import ( + archive, format_utc_iso_date, parse_iso8601_date_to_utc, redirect_to_new_route, reverse, ) +from swh.web.utils.exc import BadInputExc +from swh.web.utils.origin_visits import get_origin_visits @browse_route( diff --git a/swh/web/browse/views/release.py b/swh/web/browse/views/release.py --- a/swh/web/browse/views/release.py +++ b/swh/web/browse/views/release.py @@ -19,11 +19,10 @@ gen_release_link, gen_revision_link, ) -from swh.web.common import archive -from swh.web.common.exc import NotFoundExc, sentry_capture_exception -from swh.web.common.identifiers import get_swhids_info -from swh.web.common.typing import ReleaseMetadata, SnapshotContext, SWHObjectInfo -from swh.web.common.utils import format_utc_iso_date, reverse +from swh.web.utils import archive, format_utc_iso_date, reverse +from swh.web.utils.exc import NotFoundExc, sentry_capture_exception +from swh.web.utils.identifiers import get_swhids_info +from swh.web.utils.typing import ReleaseMetadata, SnapshotContext, SWHObjectInfo @browse_route( diff --git a/swh/web/browse/views/revision.py b/swh/web/browse/views/revision.py --- a/swh/web/browse/views/revision.py +++ b/swh/web/browse/views/revision.py @@ -28,16 +28,16 @@ prepare_content_for_display, request_content, ) -from swh.web.common import archive -from swh.web.common.exc import NotFoundExc, http_status_code_message -from swh.web.common.identifiers import get_swhids_info -from swh.web.common.typing import RevisionMetadata, SnapshotContext, SWHObjectInfo -from swh.web.common.utils import ( +from swh.web.utils import ( + archive, format_utc_iso_date, gen_path_info, reverse, swh_object_icons, ) +from swh.web.utils.exc import NotFoundExc, http_status_code_message +from swh.web.utils.identifiers import get_swhids_info +from swh.web.utils.typing import RevisionMetadata, SnapshotContext, SWHObjectInfo def _gen_content_url( diff --git a/swh/web/browse/views/snapshot.py b/swh/web/browse/views/snapshot.py --- a/swh/web/browse/views/snapshot.py +++ b/swh/web/browse/views/snapshot.py @@ -16,8 +16,8 @@ browse_snapshot_releases, get_snapshot_context, ) -from swh.web.common.exc import BadInputExc -from swh.web.common.utils import redirect_to_new_route, reverse +from swh.web.utils import redirect_to_new_route, reverse +from swh.web.utils.exc import BadInputExc def get_snapshot_from_request(request: HttpRequest) -> str: diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py deleted file mode 100644 --- a/swh/web/common/utils.py +++ /dev/null @@ -1,523 +0,0 @@ -# Copyright (C) 2017-2022 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -from datetime import datetime, timezone -import functools -import os -import re -from typing import Any, Callable, Dict, List, Mapping, Optional -import urllib.parse - -from bs4 import BeautifulSoup -from docutils.core import publish_parts -import docutils.parsers.rst -import docutils.utils -from docutils.writers.html5_polyglot import HTMLTranslator, Writer -from iso8601 import ParseError, parse_date -from pkg_resources import get_distribution -from prometheus_client.registry import CollectorRegistry -import requests -from requests.auth import HTTPBasicAuth - -from django.conf import settings -from django.core.cache import cache -from django.core.cache.backends.base import DEFAULT_TIMEOUT -from django.http import HttpRequest, QueryDict -from django.shortcuts import redirect -from django.urls import resolve -from django.urls import reverse as django_reverse - -from swh.web.auth.utils import ( - ADD_FORGE_MODERATOR_PERMISSION, - ADMIN_LIST_DEPOSIT_PERMISSION, - MAILMAP_ADMIN_PERMISSION, -) -from swh.web.common.exc import BadInputExc, sentry_capture_exception -from swh.web.config import SWH_WEB_SERVER_NAME, get_config, search - -SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) - -SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" - - -swh_object_icons = { - "alias": "mdi mdi-star", - "branch": "mdi mdi-source-branch", - "branches": "mdi mdi-source-branch", - "content": "mdi mdi-file-document", - "cnt": "mdi mdi-file-document", - "directory": "mdi mdi-folder", - "dir": "mdi mdi-folder", - "origin": "mdi mdi-source-repository", - "ori": "mdi mdi-source-repository", - "person": "mdi mdi-account", - "revisions history": "mdi mdi-history", - "release": "mdi mdi-tag", - "rel": "mdi mdi-tag", - "releases": "mdi mdi-tag", - "revision": "mdi mdi-rotate-90 mdi-source-commit", - "rev": "mdi mdi-rotate-90 mdi-source-commit", - "snapshot": "mdi mdi-camera", - "snp": "mdi mdi-camera", - "visits": "mdi mdi-calendar-month", -} - - -def reverse( - viewname: str, - url_args: Optional[Dict[str, Any]] = None, - query_params: Optional[Mapping[str, Optional[str]]] = None, - current_app: Optional[str] = None, - urlconf: Optional[str] = None, - request: Optional[HttpRequest] = None, -) -> str: - """An override of django reverse function supporting query parameters. - - Args: - viewname: the name of the django view from which to compute a url - url_args: dictionary of url arguments indexed by their names - query_params: dictionary of query parameters to append to the - reversed url - current_app: the name of the django app tighten to the view - urlconf: url configuration module - request: build an absolute URI if provided - - Returns: - str: the url of the requested view with processed arguments and - query parameters - """ - - if url_args: - url_args = {k: v for k, v in url_args.items() if v is not None} - - url = django_reverse( - viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app - ) - - params: Dict[str, str] = {} - if query_params: - params = {k: v for k, v in query_params.items() if v is not None} - - if params: - query_dict = QueryDict("", mutable=True) - query_dict.update(dict(sorted(params.items()))) - url += "?" + query_dict.urlencode(safe="/;:") - - if request is not None: - url = request.build_absolute_uri(url) - - return url - - -def datetime_to_utc(date): - """Returns datetime in UTC without timezone info - - Args: - date (datetime.datetime): input datetime with timezone info - - Returns: - datetime.datetime: datetime in UTC without timezone info - """ - if date.tzinfo and date.tzinfo != timezone.utc: - return date.astimezone(tz=timezone.utc) - else: - return date - - -def parse_iso8601_date_to_utc(iso_date: str) -> datetime: - """Given an ISO 8601 datetime string, parse the result as UTC datetime. - - Returns: - a timezone-aware datetime representing the parsed date - - Raises: - swh.web.common.exc.BadInputExc: provided date does not respect ISO 8601 format - - Samples: - - 2016-01-12 - - 2016-01-12T09:19:12+0100 - - 2007-01-14T20:34:22Z - - """ - try: - date = parse_date(iso_date) - return datetime_to_utc(date) - except ParseError as e: - raise BadInputExc(e) - - -def shorten_path(path): - """Shorten the given path: for each hash present, only return the first - 8 characters followed by an ellipsis""" - - sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}" - sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}" - - ret = re.sub(sha256_re, r"\1...", path) - return re.sub(sha1_re, r"\1...", ret) - - -def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M:%S UTC"): - """Turns a string representation of an ISO 8601 datetime string - to UTC and format it into a more human readable one. - - For instance, from the following input - string: '2017-05-04T13:27:13+02:00' the following one - is returned: '04 May 2017, 11:27 UTC'. - Custom format string may also be provided - as parameter - - Args: - iso_date (str): a string representation of an ISO 8601 date - fmt (str): optional date formatting string - - Returns: - str: a formatted string representation of the input iso date - """ - if not iso_date: - return iso_date - date = parse_iso8601_date_to_utc(iso_date) - return date.strftime(fmt) - - -def gen_path_info(path): - """Function to generate path data navigation for use - with a breadcrumb in the swh web ui. - - For instance, from a path /folder1/folder2/folder3, - it returns the following list:: - - [{'name': 'folder1', 'path': 'folder1'}, - {'name': 'folder2', 'path': 'folder1/folder2'}, - {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] - - Args: - path: a filesystem path - - Returns: - list: a list of path data for navigation as illustrated above. - - """ - path_info = [] - if path: - sub_paths = path.strip("/").split("/") - path_from_root = "" - for p in sub_paths: - path_from_root += "/" + p - path_info.append({"name": p, "path": path_from_root.strip("/")}) - return path_info - - -def parse_rst(text, report_level=2): - """ - Parse a reStructuredText string with docutils. - - Args: - text (str): string with reStructuredText markups in it - report_level (int): level of docutils report messages to print - (1 info 2 warning 3 error 4 severe 5 none) - - Returns: - docutils.nodes.document: a parsed docutils document - """ - parser = docutils.parsers.rst.Parser() - components = (docutils.parsers.rst.Parser,) - settings = docutils.frontend.OptionParser( - components=components - ).get_default_values() - settings.report_level = report_level - document = docutils.utils.new_document("rst-doc", settings=settings) - parser.parse(text, document) - return document - - -def get_client_ip(request): - """ - Return the client IP address from an incoming HTTP request. - - Args: - request (django.http.HttpRequest): the incoming HTTP request - - Returns: - str: The client IP address - """ - x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") - if x_forwarded_for: - ip = x_forwarded_for.split(",")[0] - else: - ip = request.META.get("REMOTE_ADDR") - return ip - - -def is_swh_web_development(request: HttpRequest) -> bool: - """Indicate if we are running a development version of swh-web.""" - site_base_url = request.build_absolute_uri("/") - return any( - host in site_base_url for host in ("localhost", "127.0.0.1", "testserver") - ) - - -def is_swh_web_staging(request: HttpRequest) -> bool: - """Indicate if we are running a staging version of swh-web.""" - config = get_config() - site_base_url = request.build_absolute_uri("/") - return any( - server_name in site_base_url for server_name in config["staging_server_names"] - ) - - -def is_swh_web_production(request: HttpRequest) -> bool: - """Indicate if we are running the public production version of swh-web.""" - return SWH_WEB_SERVER_NAME in request.build_absolute_uri("/") - - -browsers_supported_image_mimes = set( - [ - "image/gif", - "image/png", - "image/jpeg", - "image/bmp", - "image/webp", - "image/svg", - "image/svg+xml", - ] -) - - -def context_processor(request): - """ - Django context processor used to inject variables - in all swh-web templates. - """ - config = get_config() - if ( - hasattr(request, "user") - and request.user.is_authenticated - and not hasattr(request.user, "backend") - ): - # To avoid django.template.base.VariableDoesNotExist errors - # when rendering templates when standard Django user is logged in. - request.user.backend = "django.contrib.auth.backends.ModelBackend" - - return { - "swh_object_icons": swh_object_icons, - "available_languages": None, - "swh_client_config": config["client_config"], - "oidc_enabled": bool(config["keycloak"]["server_url"]), - "browsers_supported_image_mimes": browsers_supported_image_mimes, - "keycloak": config["keycloak"], - "site_base_url": request.build_absolute_uri("/"), - "DJANGO_SETTINGS_MODULE": os.environ["DJANGO_SETTINGS_MODULE"], - "status": config["status"], - "swh_web_dev": is_swh_web_development(request), - "swh_web_staging": is_swh_web_staging(request), - "swh_web_prod": is_swh_web_production(request), - "swh_web_version": get_distribution("swh.web").version, - "iframe_mode": False, - "ADMIN_LIST_DEPOSIT_PERMISSION": ADMIN_LIST_DEPOSIT_PERMISSION, - "ADD_FORGE_MODERATOR_PERMISSION": ADD_FORGE_MODERATOR_PERMISSION, - "MAILMAP_ADMIN_PERMISSION": MAILMAP_ADMIN_PERMISSION, - "lang": "en", - "sidebar_state": request.COOKIES.get("sidebar-state", "expanded"), - "SWH_DJANGO_APPS": settings.SWH_DJANGO_APPS, - } - - -def resolve_branch_alias( - snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] -) -> Optional[Dict[str, Any]]: - """ - Resolve branch alias in snapshot content. - - Args: - snapshot: a full snapshot content - branch: a branch alias contained in the snapshot - Returns: - The real snapshot branch that got aliased. - """ - while branch and branch["target_type"] == "alias": - if branch["target"] in snapshot["branches"]: - branch = snapshot["branches"][branch["target"]] - else: - from swh.web.common import archive - - snp = archive.lookup_snapshot( - snapshot["id"], branches_from=branch["target"], branches_count=1 - ) - if snp and branch["target"] in snp["branches"]: - branch = snp["branches"][branch["target"]] - else: - branch = None - return branch - - -class _NoHeaderHTMLTranslator(HTMLTranslator): - """ - Docutils translator subclass to customize the generation of HTML - from reST-formatted docstrings - """ - - def __init__(self, document): - super().__init__(document) - self.body_prefix = [] - self.body_suffix = [] - - -_HTML_WRITER = Writer() -_HTML_WRITER.translator_class = _NoHeaderHTMLTranslator - - -def rst_to_html(rst: str) -> str: - """ - Convert reStructuredText document into HTML. - - Args: - rst: A string containing a reStructuredText document - - Returns: - Body content of the produced HTML conversion. - - """ - settings = { - "initial_header_level": 2, - "halt_level": 4, - "traceback": True, - "file_insertion_enabled": False, - "raw_enabled": False, - } - pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) - return f'
{pp["html_body"]}
' - - -def prettify_html(html: str) -> str: - """ - Prettify an HTML document. - - Args: - html: Input HTML document - - Returns: - The prettified HTML document - """ - return BeautifulSoup(html, "lxml").prettify() - - -def django_cache( - timeout: int = DEFAULT_TIMEOUT, - catch_exception: bool = False, - exception_return_value: Any = None, - invalidate_cache_pred: Callable[[Any], bool] = lambda val: False, -): - """Decorator to put the result of a function call in Django cache, - subsequent calls will directly return the cached value. - - Args: - timeout: The number of seconds value will be hold in cache - catch_exception: If :const:`True`, any thrown exception by - the decorated function will be caught and not reraised - exception_return_value: The value to return if previous - parameter is set to :const:`True` - invalidate_cache_pred: A predicate function enabling to - invalidate the cache under certain conditions, decorated - function will then be called again - - Returns: - The returned value of the decorated function for the specified - parameters - - """ - - def inner(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - func_args = args + (0,) + tuple(sorted(kwargs.items())) - cache_key = str(hash((func.__module__, func.__name__) + func_args)) - ret = cache.get(cache_key) - if ret is None or invalidate_cache_pred(ret): - try: - ret = func(*args, **kwargs) - except Exception as exc: - if catch_exception: - sentry_capture_exception(exc) - return exception_return_value - else: - raise - else: - cache.set(cache_key, ret, timeout=timeout) - return ret - - return wrapper - - return inner - - -def _deposits_list_url( - deposits_list_base_url: str, page_size: int, username: Optional[str] -) -> str: - params = {"page_size": str(page_size)} - if username is not None: - params["username"] = username - return f"{deposits_list_base_url}?{urllib.parse.urlencode(params)}" - - -def get_deposits_list(username: Optional[str] = None) -> List[Dict[str, Any]]: - """Return the list of software deposits using swh-deposit API""" - config = get_config()["deposit"] - private_api_url = config["private_api_url"].rstrip("/") + "/" - deposits_list_base_url = private_api_url + "deposits" - deposits_list_auth = HTTPBasicAuth( - config["private_api_user"], config["private_api_password"] - ) - - deposits_list_url = _deposits_list_url( - deposits_list_base_url, page_size=1, username=username - ) - - nb_deposits = requests.get( - deposits_list_url, auth=deposits_list_auth, timeout=30 - ).json()["count"] - - @django_cache(invalidate_cache_pred=lambda data: data["count"] != nb_deposits) - def _get_deposits_data(): - deposits_list_url = _deposits_list_url( - deposits_list_base_url, page_size=nb_deposits, username=username - ) - return requests.get( - deposits_list_url, - auth=deposits_list_auth, - timeout=30, - ).json() - - deposits_data = _get_deposits_data() - - return deposits_data["results"] - - -_origin_visit_types_cache_timeout = 24 * 60 * 60 # 24 hours - - -@django_cache( - timeout=_origin_visit_types_cache_timeout, - catch_exception=True, - exception_return_value=[], -) -def origin_visit_types() -> List[str]: - """Return the exhaustive list of visit types for origins - ingested into the archive. - """ - return sorted(search().visit_types_count().keys()) - - -def redirect_to_new_route(request, new_route, permanent=True): - """Redirect a request to another route with url args and query parameters - eg: /origin//log?path=test can be redirected as - /log?url=&path=test. This can be used to deprecate routes - """ - request_path = resolve(request.path_info) - args = {**request_path.kwargs, **request.GET.dict()} - return redirect( - reverse(new_route, query_params=args), - permanent=permanent, - ) diff --git a/swh/web/inbound_email/management/commands/process_inbound_email.py b/swh/web/inbound_email/management/commands/process_inbound_email.py --- a/swh/web/inbound_email/management/commands/process_inbound_email.py +++ b/swh/web/inbound_email/management/commands/process_inbound_email.py @@ -12,8 +12,8 @@ from django.core.management.base import BaseCommand -from swh.web.common.exc import sentry_capture_exception from swh.web.inbound_email import signals +from swh.web.utils.exc import sentry_capture_exception logger = logging.getLogger(__name__) diff --git a/swh/web/misc/badges.py b/swh/web/misc/badges.py --- a/swh/web/misc/badges.py +++ b/swh/web/misc/badges.py @@ -15,10 +15,9 @@ from swh.model.exceptions import ValidationError from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.swhids import CoreSWHID, ObjectType, QualifiedSWHID -from swh.web.common import archive -from swh.web.common.exc import BadInputExc, NotFoundExc -from swh.web.common.identifiers import parse_object_type, resolve_swhid -from swh.web.common.utils import reverse +from swh.web.utils import archive, reverse +from swh.web.utils.exc import BadInputExc, NotFoundExc +from swh.web.utils.identifiers import parse_object_type, resolve_swhid _orange = "#f36a24" _blue = "#0172b2" diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py --- a/swh/web/misc/coverage.py +++ b/swh/web/misc/coverage.py @@ -15,15 +15,15 @@ from django.views.decorators.clickjacking import xframe_options_exempt from swh.scheduler.model import SchedulerMetrics -from swh.web.common import archive -from swh.web.common.utils import ( +from swh.web.config import scheduler +from swh.web.utils import ( + archive, django_cache, get_deposits_list, is_swh_web_development, is_swh_web_production, reverse, ) -from swh.web.config import scheduler _swh_arch_overview_doc = ( "https://docs.softwareheritage.org/devel/architecture/overview.html" diff --git a/swh/web/misc/iframe.py b/swh/web/misc/iframe.py --- a/swh/web/misc/iframe.py +++ b/swh/web/misc/iframe.py @@ -18,11 +18,10 @@ prepare_content_for_display, request_content, ) -from swh.web.common import archive -from swh.web.common.exc import BadInputExc, NotFoundExc, http_status_code_message -from swh.web.common.identifiers import get_swhid, get_swhids_info -from swh.web.common.typing import SnapshotContext, SWHObjectInfo -from swh.web.common.utils import gen_path_info, reverse +from swh.web.utils import archive, gen_path_info, reverse +from swh.web.utils.exc import BadInputExc, NotFoundExc, http_status_code_message +from swh.web.utils.identifiers import get_swhid, get_swhids_info +from swh.web.utils.typing import SnapshotContext, SWHObjectInfo def _get_content_rendering_data(cnt_swhid: QualifiedSWHID, path: str) -> Dict[str, Any]: diff --git a/swh/web/misc/metrics.py b/swh/web/misc/metrics.py --- a/swh/web/misc/metrics.py +++ b/swh/web/misc/metrics.py @@ -7,8 +7,8 @@ from django.http import HttpResponse -from swh.web.common.utils import SWH_WEB_METRICS_REGISTRY from swh.web.save_code_now.origin_save import compute_save_requests_metrics +from swh.web.utils import SWH_WEB_METRICS_REGISTRY def prometheus_metrics(request): diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -14,10 +14,10 @@ from django.urls import re_path as url from django.views.decorators.clickjacking import xframe_options_exempt -from swh.web.common import archive -from swh.web.common.exc import sentry_capture_exception from swh.web.config import get_config from swh.web.misc.metrics import prometheus_metrics +from swh.web.utils import archive +from swh.web.utils.exc import sentry_capture_exception def _jslicenses(request): diff --git a/swh/web/save_code_now/models.py b/swh/web/save_code_now/models.py --- a/swh/web/save_code_now/models.py +++ b/swh/web/save_code_now/models.py @@ -5,7 +5,7 @@ from django.db import models -from swh.web.common.typing import SaveOriginRequestInfo +from swh.web.utils.typing import SaveOriginRequestInfo class SaveAuthorizedOrigin(models.Model): diff --git a/swh/web/save_code_now/origin_save.py b/swh/web/save_code_now/origin_save.py --- a/swh/web/save_code_now/origin_save.py +++ b/swh/web/save_code_now/origin_save.py @@ -20,15 +20,6 @@ from django.utils.html import escape from swh.scheduler.utils import create_oneshot_task_dict -from swh.web.common import archive -from swh.web.common.exc import ( - BadInputExc, - ForbiddenExc, - NotFoundExc, - sentry_capture_exception, -) -from swh.web.common.typing import OriginExistenceCheckInfo, SaveOriginRequestInfo -from swh.web.common.utils import SWH_WEB_METRICS_REGISTRY, parse_iso8601_date_to_utc from swh.web.config import get_config, scheduler from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, @@ -46,6 +37,14 @@ SaveOriginRequest, SaveUnauthorizedOrigin, ) +from swh.web.utils import SWH_WEB_METRICS_REGISTRY, archive, parse_iso8601_date_to_utc +from swh.web.utils.exc import ( + BadInputExc, + ForbiddenExc, + NotFoundExc, + sentry_capture_exception, +) +from swh.web.utils.typing import OriginExistenceCheckInfo, SaveOriginRequestInfo logger = logging.getLogger(__name__) @@ -676,7 +675,7 @@ Raises: BadInputExc: the visit type or origin url is invalid - swh.web.common.exc.NotFoundExc: no save requests can be found for the + swh.web.utils.exc.NotFoundExc: no save requests can be found for the given origin Returns: diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -15,7 +15,6 @@ from django.utils import encoding -from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID from swh.web.config import get_config # Fix django-js-reverse 0.9.1 compatibility with django 4.x @@ -46,7 +45,7 @@ SWH_BASE_DJANGO_APPS = [ "swh.web.auth", "swh.web.browse", - "swh.web.common", + "swh.web.utils", "swh.web.api", ] SWH_EXTRA_DJANGO_APPS = [ @@ -82,8 +81,8 @@ "swh.auth.django.middlewares.OIDCSessionExpiredMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", - "swh.web.common.middlewares.ThrottlingHeadersMiddleware", - "swh.web.common.middlewares.ExceptionMiddleware", + "swh.web.utils.middlewares.ThrottlingHeadersMiddleware", + "swh.web.utils.middlewares.ExceptionMiddleware", ] # Compress all assets (static ones and dynamically generated html) @@ -119,10 +118,10 @@ "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", - "swh.web.common.utils.context_processor", + "swh.web.utils.context_processor", ], "libraries": { - "swh_templatetags": "swh.web.common.swh_templatetags", + "swh_templatetags": "swh.web.utils.swh_templatetags", }, }, }, @@ -341,6 +340,7 @@ "swh.auth.django.backends.OIDCAuthorizationCodePKCEBackend", ] +OIDC_SWH_WEB_CLIENT_ID = "swh-web" SWH_AUTH_SERVER_URL = swh_web_config["keycloak"]["server_url"] SWH_AUTH_REALM_NAME = swh_web_config["keycloak"]["realm_name"] SWH_AUTH_CLIENT_ID = OIDC_SWH_WEB_CLIENT_ID diff --git a/swh/web/settings/development.py b/swh/web/settings/development.py --- a/swh/web/settings/development.py +++ b/swh/web/settings/development.py @@ -12,7 +12,7 @@ from .common import * # noqa from .common import MIDDLEWARE -MIDDLEWARE += ["swh.web.common.middlewares.HtmlPrettifyMiddleware"] +MIDDLEWARE += ["swh.web.utils.middlewares.HtmlPrettifyMiddleware"] AUTH_PASSWORD_VALIDATORS = [] # disable any pwd validation mechanism diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -18,7 +18,7 @@ from .common import * # noqa MIDDLEWARE += [ - "swh.web.common.middlewares.HtmlMinifyMiddleware", + "swh.web.utils.middlewares.HtmlMinifyMiddleware", ] if swh_web_config.get("throttling", {}).get("cache_uri"): diff --git a/swh/web/tests/add_forge_now/test_api_views.py b/swh/web/tests/add_forge_now/test_api_views.py --- a/swh/web/tests/add_forge_now/test_api_views.py +++ b/swh/web/tests/add_forge_now/test_api_views.py @@ -14,15 +14,15 @@ import pytest from swh.web.add_forge_now.models import Request, RequestHistory -from swh.web.common.utils import reverse from swh.web.config import get_config from swh.web.inbound_email.utils import get_address_for_pk -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_get_responses, check_api_post_response, check_http_get_response, check_http_post_response, ) +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/add_forge_now/test_app.py b/swh/web/tests/add_forge_now/test_app.py --- a/swh/web/tests/add_forge_now/test_app.py +++ b/swh/web/tests/add_forge_now/test_app.py @@ -8,9 +8,9 @@ from django.urls import get_resolver from swh.web.add_forge_now.urls import urlpatterns -from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_not_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/add_forge_now/test_views.py b/swh/web/tests/add_forge_now/test_views.py --- a/swh/web/tests/add_forge_now/test_views.py +++ b/swh/web/tests/add_forge_now/test_views.py @@ -7,8 +7,8 @@ import pytest -from swh.web.common.utils import reverse -from swh.web.tests.utils import check_http_get_response +from swh.web.tests.helpers import check_http_get_response +from swh.web.utils import reverse from .test_api_views import create_add_forge_request diff --git a/swh/web/tests/api/test_api_lookup.py b/swh/web/tests/api/test_api_lookup.py --- a/swh/web/tests/api/test_api_lookup.py +++ b/swh/web/tests/api/test_api_lookup.py @@ -6,7 +6,7 @@ import pytest from swh.web.api.views import utils -from swh.web.common.exc import NotFoundExc +from swh.web.utils.exc import NotFoundExc def test_genericapi_lookup_nothing_is_found(): diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py --- a/swh/web/tests/api/test_apidoc.py +++ b/swh/web/tests/api/test_apidoc.py @@ -12,9 +12,9 @@ from swh.storage.exc import StorageAPIError, StorageDBError from swh.web.api.apidoc import _parse_httpdomain_doc, api_doc from swh.web.api.apiurls import api_route -from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc -from swh.web.common.utils import prettify_html, reverse -from swh.web.tests.utils import check_api_get_responses, check_html_get_response +from swh.web.tests.helpers import check_api_get_responses, check_html_get_response +from swh.web.utils import prettify_html, reverse +from swh.web.utils.exc import BadInputExc, ForbiddenExc, NotFoundExc _httpdomain_doc = """ .. http:get:: /api/1/revision/(sha1_git)/ diff --git a/swh/web/tests/api/test_apiresponse.py b/swh/web/tests/api/test_apiresponse.py --- a/swh/web/tests/api/test_apiresponse.py +++ b/swh/web/tests/api/test_apiresponse.py @@ -18,10 +18,10 @@ make_api_response, transform, ) -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import check_http_get_response, check_http_post_response +from swh.web.tests.helpers import check_http_get_response, check_http_post_response +from swh.web.utils import reverse +from swh.web.utils.identifiers import gen_swhid def test_compute_link_header(): diff --git a/swh/web/tests/api/test_apiurls.py b/swh/web/tests/api/test_apiurls.py --- a/swh/web/tests/api/test_apiurls.py +++ b/swh/web/tests/api/test_apiurls.py @@ -5,8 +5,8 @@ from swh.web.api.apiurls import api_route -from swh.web.common.utils import reverse -from swh.web.tests.utils import check_api_get_responses +from swh.web.tests.helpers import check_api_get_responses +from swh.web.utils import reverse @api_route(r"/some/route/(?P[0-9]+)/", "api-1-some-route") diff --git a/swh/web/tests/api/test_throttling.py b/swh/web/tests/api/test_throttling.py --- a/swh/web/tests/api/test_throttling.py +++ b/swh/web/tests/api/test_throttling.py @@ -25,7 +25,7 @@ scope3_limiter_rate, scope3_limiter_rate_post, ) -from swh.web.tests.utils import create_django_permission +from swh.web.tests.helpers import create_django_permission from swh.web.urls import urlpatterns diff --git a/swh/web/tests/api/test_utils.py b/swh/web/tests/api/test_utils.py --- a/swh/web/tests/api/test_utils.py +++ b/swh/web/tests/api/test_utils.py @@ -8,8 +8,8 @@ from swh.model.hashutil import DEFAULT_ALGORITHMS from swh.model.model import Origin from swh.web.api import utils -from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.utils import resolve_branch_alias, reverse +from swh.web.utils import resolve_branch_alias, reverse +from swh.web.utils.origin_visits import get_origin_visits url_map = [ { diff --git a/swh/web/tests/api/views/test_content.py b/swh/web/tests/api/views/test_content.py --- a/swh/web/tests/api/views/test_content.py +++ b/swh/web/tests/api/views/test_content.py @@ -5,14 +5,14 @@ import pytest -from swh.web.common.utils import reverse from swh.web.tests.conftest import fossology_missing from swh.web.tests.data import random_content -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_get_responses, check_api_post_responses, check_http_get_response, ) +from swh.web.utils import reverse def test_api_content_filetype(api_client, indexer_data, content): diff --git a/swh/web/tests/api/views/test_directory.py b/swh/web/tests/api/views/test_directory.py --- a/swh/web/tests/api/views/test_directory.py +++ b/swh/web/tests/api/views/test_directory.py @@ -6,9 +6,9 @@ import random from swh.web.api.utils import enrich_directory_entry -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.utils import check_api_get_responses, check_http_get_response +from swh.web.tests.helpers import check_api_get_responses, check_http_get_response +from swh.web.utils import reverse def test_api_directory(api_client, archive_data, directory): diff --git a/swh/web/tests/api/views/test_graph.py b/swh/web/tests/api/views/test_graph.py --- a/swh/web/tests/api/views/test_graph.py +++ b/swh/web/tests/api/views/test_graph.py @@ -15,9 +15,9 @@ from swh.model.hashutil import hash_to_bytes from swh.model.swhids import ExtendedObjectType, ExtendedSWHID from swh.web.api.views.graph import API_GRAPH_PERM -from swh.web.common.utils import reverse from swh.web.config import SWH_WEB_INTERNAL_SERVER_NAME, get_config -from swh.web.tests.utils import check_http_get_response +from swh.web.tests.helpers import check_http_get_response +from swh.web.utils import reverse def test_graph_endpoint_no_authentication_for_vpn_users(api_client, requests_mock): diff --git a/swh/web/tests/api/views/test_identifiers.py b/swh/web/tests/api/views/test_identifiers.py --- a/swh/web/tests/api/views/test_identifiers.py +++ b/swh/web/tests/api/views/test_identifiers.py @@ -5,10 +5,10 @@ from swh.model.swhids import ObjectType -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.utils import check_api_get_responses, check_api_post_responses +from swh.web.tests.helpers import check_api_get_responses, check_api_post_responses +from swh.web.utils import reverse +from swh.web.utils.identifiers import gen_swhid def test_swhid_resolve_success( diff --git a/swh/web/tests/api/views/test_metadata.py b/swh/web/tests/api/views/test_metadata.py --- a/swh/web/tests/api/views/test_metadata.py +++ b/swh/web/tests/api/views/test_metadata.py @@ -10,9 +10,9 @@ from swh.model.hypothesis_strategies import raw_extrinsic_metadata from swh.model.model import Origin -from swh.web.common.utils import reverse from swh.web.tests.api.views.utils import scroll_results -from swh.web.tests.utils import check_api_get_responses, check_http_get_response +from swh.web.tests.helpers import check_api_get_responses, check_http_get_response +from swh.web.utils import reverse @given(raw_extrinsic_metadata()) diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -17,9 +17,6 @@ from swh.storage.exc import StorageAPIError, StorageDBError from swh.storage.utils import now from swh.web.api.utils import enrich_origin, enrich_origin_visit -from swh.web.common.exc import BadInputExc -from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.utils import reverse from swh.web.tests.api.views.utils import scroll_results from swh.web.tests.data import ( INDEXER_TOOL, @@ -28,8 +25,11 @@ ORIGIN_METADATA_KEY, ORIGIN_METADATA_VALUE, ) +from swh.web.tests.helpers import check_api_get_responses from swh.web.tests.strategies import new_origin, new_snapshots, visit_dates -from swh.web.tests.utils import check_api_get_responses +from swh.web.utils import reverse +from swh.web.utils.exc import BadInputExc +from swh.web.utils.origin_visits import get_origin_visits def test_api_lookup_origin_visits_raise_error(api_client, origin, mocker): @@ -452,7 +452,7 @@ def test_api_origin_search(api_client, mocker, backend): if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch("swh.web.common.archive.search", None) + mocker.patch("swh.web.utils.archive.search", None) expected_origins = { "https://github.com/wcoder/highlightjs-line-numbers.js", @@ -504,7 +504,7 @@ def test_api_origin_search_words(api_client, mocker, backend): if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch("swh.web.common.archive.search", None) + mocker.patch("swh.web.utils.archive.search", None) expected_origins = { "https://github.com/wcoder/highlightjs-line-numbers.js", @@ -554,7 +554,7 @@ def test_api_origin_search_visit_type(api_client, mocker, backend): if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch("swh.web.common.archive.search", None) + mocker.patch("swh.web.utils.archive.search", None) expected_origins = { "https://github.com/wcoder/highlightjs-line-numbers.js", @@ -590,7 +590,7 @@ ORIGINS = [{"url": origin} for origin in expected_origins] - mock_archive_search = mocker.patch("swh.web.common.archive.search") + mock_archive_search = mocker.patch("swh.web.utils.archive.search") mock_archive_search.origin_search.return_value = PagedResult( results=ORIGINS, next_page_token=None, @@ -612,7 +612,7 @@ def test_api_origin_search_ql_syntax_error(api_client, mocker): - mock_archive_search = mocker.patch("swh.web.common.archive.search") + mock_archive_search = mocker.patch("swh.web.utils.archive.search") mock_archive_search.origin_search.side_effect = SearchQuerySyntaxError( "Invalid syntax" ) @@ -641,7 +641,7 @@ if backend != "swh-search": # equivalent to not configuring search in the config - mocker.patch("swh.web.common.archive.search", None) + mocker.patch("swh.web.utils.archive.search", None) expected_origins = { "https://github.com/wcoder/highlightjs-line-numbers.js", @@ -667,7 +667,7 @@ ) else: # equivalent to not configuring search in the config - mocker.patch("swh.web.common.archive.search", None) + mocker.patch("swh.web.utils.archive.search", None) archive_data.origin_add( [Origin(url="http://foobar/{}".format(i)) for i in range(2000)] @@ -685,7 +685,7 @@ @pytest.mark.parametrize("backend", ["swh-search", "swh-indexer-storage"]) def test_api_origin_metadata_search(api_client, mocker, backend): - mock_config = mocker.patch("swh.web.common.archive.config") + mock_config = mocker.patch("swh.web.utils.archive.config") mock_config.get_config.return_value = { "search_config": {"metadata_backend": backend} } @@ -729,7 +729,7 @@ def test_api_origin_metadata_search_limit(api_client, mocker): - mock_idx_storage = mocker.patch("swh.web.common.archive.idx_storage") + mock_idx_storage = mocker.patch("swh.web.utils.archive.idx_storage") oimsft = mock_idx_storage.origin_intrinsic_metadata_search_fulltext oimsft.side_effect = lambda conjunction, limit: [ @@ -779,7 +779,7 @@ def test_api_origin_metadata_search_invalid(api_client, mocker): - mock_idx_storage = mocker.patch("swh.web.common.archive.idx_storage") + mock_idx_storage = mocker.patch("swh.web.utils.archive.idx_storage") url = reverse("api-1-origin-metadata-search") check_api_get_responses(api_client, url, status_code=400) mock_idx_storage.assert_not_called() @@ -788,7 +788,7 @@ @pytest.mark.parametrize("backend", ["swh-counters", "swh-storage"]) def test_api_stat_counters(api_client, mocker, backend): - mock_config = mocker.patch("swh.web.common.archive.config") + mock_config = mocker.patch("swh.web.utils.archive.config") mock_config.get_config.return_value = {"counters_backend": backend} url = reverse("api-1-stat-counters") diff --git a/swh/web/tests/api/views/test_ping.py b/swh/web/tests/api/views/test_ping.py --- a/swh/web/tests/api/views/test_ping.py +++ b/swh/web/tests/api/views/test_ping.py @@ -3,8 +3,8 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.web.common.utils import reverse -from swh.web.tests.utils import check_api_get_responses +from swh.web.tests.helpers import check_api_get_responses +from swh.web.utils import reverse def test_api_1_ping(api_client): diff --git a/swh/web/tests/api/views/test_raw.py b/swh/web/tests/api/views/test_raw.py --- a/swh/web/tests/api/views/test_raw.py +++ b/swh/web/tests/api/views/test_raw.py @@ -10,13 +10,13 @@ from swh.model.hashutil import hash_to_bytes from swh.web.api.throttling import SwhWebUserRateThrottle from swh.web.auth.utils import API_RAW_OBJECT_PERMISSION -from swh.web.common.utils import reverse from swh.web.settings.tests import api_raw_object_rate -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_get_responses, check_http_get_response, create_django_permission, ) +from swh.web.utils import reverse @pytest.fixture diff --git a/swh/web/tests/api/views/test_release.py b/swh/web/tests/api/views/test_release.py --- a/swh/web/tests/api/views/test_release.py +++ b/swh/web/tests/api/views/test_release.py @@ -7,9 +7,9 @@ from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.model import ObjectType, Person, Release, TimestampWithTimezone -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 -from swh.web.tests.utils import check_api_get_responses, check_http_get_response +from swh.web.tests.helpers import check_api_get_responses, check_http_get_response +from swh.web.utils import reverse def test_api_release(api_client, archive_data, release): diff --git a/swh/web/tests/api/views/test_revision.py b/swh/web/tests/api/views/test_revision.py --- a/swh/web/tests/api/views/test_revision.py +++ b/swh/web/tests/api/views/test_revision.py @@ -15,10 +15,10 @@ TimestampWithTimezone, ) from swh.web.api.utils import enrich_content, enrich_directory_entry, enrich_revision -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 +from swh.web.tests.helpers import check_api_get_responses, check_http_get_response from swh.web.tests.strategies import new_person, new_swh_date -from swh.web.tests.utils import check_api_get_responses, check_http_get_response +from swh.web.utils import reverse def test_api_revision(api_client, archive_data, revision): diff --git a/swh/web/tests/api/views/test_snapshot.py b/swh/web/tests/api/views/test_snapshot.py --- a/swh/web/tests/api/views/test_snapshot.py +++ b/swh/web/tests/api/views/test_snapshot.py @@ -10,10 +10,10 @@ from swh.model.hashutil import hash_to_hex from swh.model.model import Snapshot from swh.web.api.utils import enrich_snapshot -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 +from swh.web.tests.helpers import check_api_get_responses, check_http_get_response from swh.web.tests.strategies import new_snapshot -from swh.web.tests.utils import check_api_get_responses, check_http_get_response +from swh.web.utils import reverse def test_api_snapshot(api_client, archive_data, snapshot): diff --git a/swh/web/tests/api/views/test_stat.py b/swh/web/tests/api/views/test_stat.py --- a/swh/web/tests/api/views/test_stat.py +++ b/swh/web/tests/api/views/test_stat.py @@ -4,9 +4,9 @@ # See top-level LICENSE file for more information from swh.storage.exc import StorageAPIError, StorageDBError -from swh.web.common.exc import BadInputExc -from swh.web.common.utils import reverse -from swh.web.tests.utils import check_api_get_responses +from swh.web.tests.helpers import check_api_get_responses +from swh.web.utils import reverse +from swh.web.utils.exc import BadInputExc def test_api_1_stat_counters_raise_error(api_client, mocker): diff --git a/swh/web/tests/api/views/test_vault.py b/swh/web/tests/api/views/test_vault.py --- a/swh/web/tests/api/views/test_vault.py +++ b/swh/web/tests/api/views/test_vault.py @@ -9,13 +9,13 @@ from swh.model.swhids import CoreSWHID from swh.vault.exc import NotFoundExc -from swh.web.common.utils import reverse -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_get_responses, check_api_post_responses, check_http_get_response, check_http_post_response, ) +from swh.web.utils import reverse ##################### # Current API: @@ -73,7 +73,7 @@ def test_api_vault_cook_notfound( api_client, mocker, directory, revision, unknown_directory, unknown_revision ): - mock_vault = mocker.patch("swh.web.common.archive.vault") + mock_vault = mocker.patch("swh.web.utils.archive.vault") mock_vault.cook.side_effect = NotFoundExc("object not found") mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found") mock_vault.progress.side_effect = NotFoundExc("cooking request not found") @@ -273,7 +273,7 @@ def test_api_vault_cook_notfound_legacy( api_client, mocker, directory, revision, unknown_directory, unknown_revision ): - mock_vault = mocker.patch("swh.web.common.archive.vault") + mock_vault = mocker.patch("swh.web.utils.archive.vault") mock_vault.cook.side_effect = NotFoundExc("object not found") mock_vault.fetch.side_effect = NotFoundExc("cooked archive not found") mock_vault.progress.side_effect = NotFoundExc("cooking request not found") diff --git a/swh/web/tests/api/views/utils.py b/swh/web/tests/api/views/utils.py --- a/swh/web/tests/api/views/utils.py +++ b/swh/web/tests/api/views/utils.py @@ -5,7 +5,7 @@ from requests.utils import parse_header_links -from swh.web.tests.utils import check_api_get_responses +from swh.web.tests.helpers import check_api_get_responses def scroll_results(api_client, url): diff --git a/swh/web/tests/auth/test_views.py b/swh/web/tests/auth/test_views.py --- a/swh/web/tests/auth/test_views.py +++ b/swh/web/tests/auth/test_views.py @@ -9,20 +9,21 @@ import pytest +from django.conf import settings from django.http import QueryDict from swh.auth.keycloak import KeycloakError from swh.web.auth.models import OIDCUserOfflineTokens -from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID, decrypt_data -from swh.web.common.utils import reverse +from swh.web.auth.utils import decrypt_data from swh.web.config import get_config from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_html_get_response, check_http_get_response, check_http_post_response, ) from swh.web.urls import _default_view as homepage_view +from swh.web.utils import reverse def _check_oidc_login_code_flow_data( @@ -36,7 +37,7 @@ # check redirect url is valid assert urljoin(response["location"], parsed_url.path) == authorization_url assert "client_id" in query_dict - assert query_dict["client_id"] == OIDC_SWH_WEB_CLIENT_ID + assert query_dict["client_id"] == settings.OIDC_SWH_WEB_CLIENT_ID assert "response_type" in query_dict assert query_dict["response_type"] == "code" assert "redirect_uri" in query_dict diff --git a/swh/web/tests/browse/test_snapshot_context.py b/swh/web/tests/browse/test_snapshot_context.py --- a/swh/web/tests/browse/test_snapshot_context.py +++ b/swh/web/tests/browse/test_snapshot_context.py @@ -16,14 +16,14 @@ get_snapshot_context, ) from swh.web.browse.utils import gen_revision_url -from swh.web.common.identifiers import gen_swhid -from swh.web.common.origin_visits import get_origin_visit, get_origin_visits -from swh.web.common.typing import ( +from swh.web.utils import format_utc_iso_date, reverse +from swh.web.utils.identifiers import gen_swhid +from swh.web.utils.origin_visits import get_origin_visit, get_origin_visits +from swh.web.utils.typing import ( SnapshotBranchInfo, SnapshotContext, SnapshotReleaseInfo, ) -from swh.web.common.utils import format_utc_iso_date, reverse def test_get_origin_visit_snapshot_simple(archive_data, origin_with_multiple_visits): diff --git a/swh/web/tests/browse/test_utils.py b/swh/web/tests/browse/test_utils.py --- a/swh/web/tests/browse/test_utils.py +++ b/swh/web/tests/browse/test_utils.py @@ -17,8 +17,8 @@ prepare_content_for_display, re_encode_content, ) -from swh.web.common.utils import reverse from swh.web.tests.data import get_tests_data +from swh.web.utils import reverse def test_get_mimetype_and_encoding_for_content(): diff --git a/swh/web/tests/browse/views/test_content.py b/swh/web/tests/browse/views/test_content.py --- a/swh/web/tests/browse/views/test_content.py +++ b/swh/web/tests/browse/views/test_content.py @@ -20,17 +20,17 @@ prepare_content_for_display, re_encode_content, ) -from swh.web.common.exc import NotFoundExc -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import ( +from swh.web.tests.data import get_content +from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response, check_http_get_response +from swh.web.utils import ( format_utc_iso_date, gen_path_info, parse_iso8601_date_to_utc, reverse, ) -from swh.web.tests.data import get_content -from swh.web.tests.django_asserts import assert_contains, assert_not_contains -from swh.web.tests.utils import check_html_get_response, check_http_get_response +from swh.web.utils.exc import NotFoundExc +from swh.web.utils.identifiers import gen_swhid def test_content_view_text(client, archive_data, content_text): @@ -752,10 +752,10 @@ def test_browse_origin_content_no_visit(client, mocker, origin): mock_get_origin_visits = mocker.patch( - "swh.web.common.origin_visits.get_origin_visits" + "swh.web.utils.origin_visits.get_origin_visits" ) mock_get_origin_visits.return_value = [] - mock_archive = mocker.patch("swh.web.common.origin_visits.archive") + mock_archive = mocker.patch("swh.web.utils.origin_visits.archive") mock_archive.lookup_origin_visit_latest.return_value = None url = reverse( "browse-content", @@ -771,7 +771,7 @@ def test_browse_origin_content_unknown_visit(client, mocker, origin): mock_get_origin_visits = mocker.patch( - "swh.web.common.origin_visits.get_origin_visits" + "swh.web.utils.origin_visits.get_origin_visits" ) mock_get_origin_visits.return_value = [{"visit": 1}] diff --git a/swh/web/tests/browse/views/test_directory.py b/swh/web/tests/browse/views/test_directory.py --- a/swh/web/tests/browse/views/test_directory.py +++ b/swh/web/tests/browse/views/test_directory.py @@ -29,11 +29,11 @@ from swh.model.swhids import ObjectType from swh.storage.utils import now from swh.web.browse.snapshot_context import process_snapshot_branches -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import gen_path_info, reverse from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response from swh.web.tests.strategies import new_person, new_swh_date -from swh.web.tests.utils import check_html_get_response +from swh.web.utils import gen_path_info, reverse +from swh.web.utils.identifiers import gen_swhid def test_root_directory_view(client, archive_data, directory): diff --git a/swh/web/tests/browse/views/test_identifiers.py b/swh/web/tests/browse/views/test_identifiers.py --- a/swh/web/tests/browse/views/test_identifiers.py +++ b/swh/web/tests/browse/views/test_identifiers.py @@ -8,10 +8,10 @@ from swh.model.model import Origin from swh.model.swhids import ObjectType -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse +from swh.web.utils.identifiers import gen_swhid def test_content_id_browse(client, content): diff --git a/swh/web/tests/browse/views/test_origin.py b/swh/web/tests/browse/views/test_origin.py --- a/swh/web/tests/browse/views/test_origin.py +++ b/swh/web/tests/browse/views/test_origin.py @@ -22,12 +22,12 @@ from swh.model.swhids import ObjectType from swh.storage.utils import now from swh.web.browse.snapshot_context import process_snapshot_branches -from swh.web.common.exc import NotFoundExc -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import format_utc_iso_date, parse_iso8601_date_to_utc, reverse from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response from swh.web.tests.strategies import new_origin, new_snapshot, visit_dates -from swh.web.tests.utils import check_html_get_response +from swh.web.utils import format_utc_iso_date, parse_iso8601_date_to_utc, reverse +from swh.web.utils.exc import NotFoundExc +from swh.web.utils.identifiers import gen_swhid def test_origin_visits_browse(client, archive_data, origin_with_multiple_visits): @@ -452,10 +452,10 @@ def test_browse_origin_directory_no_visit(client, mocker, origin): mock_get_origin_visits = mocker.patch( - "swh.web.common.origin_visits.get_origin_visits" + "swh.web.utils.origin_visits.get_origin_visits" ) mock_get_origin_visits.return_value = [] - mock_archive = mocker.patch("swh.web.common.origin_visits.archive") + mock_archive = mocker.patch("swh.web.utils.origin_visits.archive") mock_archive.lookup_origin_visit_latest.return_value = None url = reverse("browse-origin-directory", query_params={"origin_url": origin["url"]}) diff --git a/swh/web/tests/browse/views/test_release.py b/swh/web/tests/browse/views/test_release.py --- a/swh/web/tests/browse/views/test_release.py +++ b/swh/web/tests/browse/views/test_release.py @@ -8,10 +8,10 @@ from django.utils.html import escape from swh.model.swhids import ObjectType -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import format_utc_iso_date, reverse from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import format_utc_iso_date, reverse +from swh.web.utils.identifiers import gen_swhid def test_release_browse(client, archive_data, release): diff --git a/swh/web/tests/browse/views/test_revision.py b/swh/web/tests/browse/views/test_revision.py --- a/swh/web/tests/browse/views/test_revision.py +++ b/swh/web/tests/browse/views/test_revision.py @@ -13,11 +13,11 @@ from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.model import Revision, RevisionType, TimestampWithTimezone from swh.model.swhids import ObjectType -from swh.web.common.identifiers import gen_swhid -from swh.web.common.utils import format_utc_iso_date, parse_iso8601_date_to_utc, reverse from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response from swh.web.tests.strategies import new_origin, new_person, new_swh_date -from swh.web.tests.utils import check_html_get_response +from swh.web.utils import format_utc_iso_date, parse_iso8601_date_to_utc, reverse +from swh.web.utils.identifiers import gen_swhid def test_revision_browse(client, archive_data, revision): diff --git a/swh/web/tests/browse/views/test_snapshot.py b/swh/web/tests/browse/views/test_snapshot.py --- a/swh/web/tests/browse/views/test_snapshot.py +++ b/swh/web/tests/browse/views/test_snapshot.py @@ -28,11 +28,11 @@ ) from swh.storage.utils import now from swh.web.browse.snapshot_context import process_snapshot_branches -from swh.web.common.utils import reverse from swh.web.tests.data import random_sha1 from swh.web.tests.django_asserts import assert_contains, assert_not_contains +from swh.web.tests.helpers import check_html_get_response from swh.web.tests.strategies import new_origin, new_person, new_swh_date, visit_dates -from swh.web.tests.utils import check_html_get_response +from swh.web.utils import reverse @pytest.mark.parametrize( diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -21,6 +21,7 @@ import pytest from pytest_django.fixtures import SettingsWrapper +from django.conf import settings from django.contrib.auth.models import User from django.core.cache import cache from django.test.utils import setup_databases @@ -43,11 +44,7 @@ ADD_FORGE_MODERATOR_PERMISSION, MAILMAP_ADMIN_PERMISSION, MAILMAP_PERMISSION, - OIDC_SWH_WEB_CLIENT_ID, ) -from swh.web.common import converters -from swh.web.common.typing import OriginVisitInfo -from swh.web.common.utils import browsers_supported_image_mimes from swh.web.config import get_config from swh.web.save_code_now.origin_save import get_scheduler_load_task_types from swh.web.tests.data import ( @@ -58,7 +55,9 @@ random_sha1_bytes, random_sha256, ) -from swh.web.tests.utils import create_django_permission +from swh.web.tests.helpers import create_django_permission +from swh.web.utils import browsers_supported_image_mimes, converters +from swh.web.utils.typing import OriginVisitInfo os.environ["LC_ALL"] = "C.UTF-8" @@ -1024,7 +1023,7 @@ keycloak_oidc.server_url = keycloak_config["server_url"] keycloak_oidc.realm_name = keycloak_config["realm_name"] - keycloak_oidc.client_id = OIDC_SWH_WEB_CLIENT_ID + keycloak_oidc.client_id = settings.OIDC_SWH_WEB_CLIENT_ID keycloak_oidc_client = mocker.patch("swh.web.auth.views.keycloak_oidc_client") keycloak_oidc_client.return_value = keycloak_oidc diff --git a/swh/web/tests/create_test_users.py b/swh/web/tests/create_test_users.py --- a/swh/web/tests/create_test_users.py +++ b/swh/web/tests/create_test_users.py @@ -13,7 +13,7 @@ MAILMAP_ADMIN_PERMISSION, SWH_AMBASSADOR_PERMISSION, ) -from swh.web.tests.utils import create_django_permission +from swh.web.tests.helpers import create_django_permission User = get_user_model() diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -40,7 +40,7 @@ prepare_content_for_display, re_encode_content, ) -from swh.web.common import archive +from swh.web.utils import archive # Module used to initialize data that will be provided as tests input diff --git a/swh/web/tests/deposit/test_app.py b/swh/web/tests/deposit/test_app.py --- a/swh/web/tests/deposit/test_app.py +++ b/swh/web/tests/deposit/test_app.py @@ -7,10 +7,10 @@ from django.urls import get_resolver -from swh.web.common.utils import reverse from swh.web.deposit.urls import urlpatterns from swh.web.tests.django_asserts import assert_not_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/deposit/test_views.py b/swh/web/tests/deposit/test_views.py --- a/swh/web/tests/deposit/test_views.py +++ b/swh/web/tests/deposit/test_views.py @@ -8,13 +8,13 @@ import pytest from swh.web.auth.utils import ADMIN_LIST_DEPOSIT_PERMISSION -from swh.web.common.utils import reverse from swh.web.config import get_config -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_html_get_response, check_http_get_response, create_django_permission, ) +from swh.web.utils import reverse def test_deposit_admin_view_not_available_for_anonymous_user(client): diff --git a/swh/web/tests/utils.py b/swh/web/tests/helpers.py rename from swh/web/tests/utils.py rename to swh/web/tests/helpers.py diff --git a/swh/web/tests/mailmap/test_app.py b/swh/web/tests/mailmap/test_app.py --- a/swh/web/tests/mailmap/test_app.py +++ b/swh/web/tests/mailmap/test_app.py @@ -7,10 +7,10 @@ from django.urls import get_resolver -from swh.web.common.utils import reverse from swh.web.mailmap.urls import urlpatterns from swh.web.tests.django_asserts import assert_not_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/mailmap/test_mailmap.py b/swh/web/tests/mailmap/test_mailmap.py --- a/swh/web/tests/mailmap/test_mailmap.py +++ b/swh/web/tests/mailmap/test_mailmap.py @@ -15,13 +15,13 @@ from django.db import transaction from swh.model.model import Person -from swh.web.common.utils import reverse from swh.web.mailmap.models import UserMailmap, UserMailmapEvent -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_post_response, check_http_get_response, check_http_post_response, ) +from swh.web.utils import reverse @pytest.mark.django_db(transaction=True) diff --git a/swh/web/tests/misc/test_badges.py b/swh/web/tests/misc/test_badges.py --- a/swh/web/tests/misc/test_badges.py +++ b/swh/web/tests/misc/test_badges.py @@ -8,13 +8,12 @@ from swh.model.hashutil import hash_to_bytes from swh.model.swhids import ObjectType, QualifiedSWHID -from swh.web.common import archive -from swh.web.common.identifiers import resolve_swhid -from swh.web.common.utils import reverse from swh.web.misc.badges import _badge_config, _get_logo_data from swh.web.tests.django_asserts import assert_contains +from swh.web.tests.helpers import check_http_get_response from swh.web.tests.strategies import new_origin -from swh.web.tests.utils import check_http_get_response +from swh.web.utils import archive, reverse +from swh.web.utils.identifiers import resolve_swhid def test_content_badge(client, content): diff --git a/swh/web/tests/misc/test_coverage.py b/swh/web/tests/misc/test_coverage.py --- a/swh/web/tests/misc/test_coverage.py +++ b/swh/web/tests/misc/test_coverage.py @@ -16,11 +16,11 @@ from django.utils.html import escape from swh.scheduler.model import LastVisitStatus, ListedOrigin, OriginVisitStats -from swh.web.common.utils import reverse from swh.web.config import SWH_WEB_SERVER_NAME from swh.web.misc.coverage import deposited_origins, legacy_origins, listed_origins from swh.web.tests.django_asserts import assert_contains, assert_not_contains -from swh.web.tests.utils import check_html_get_response, check_http_get_response +from swh.web.tests.helpers import check_html_get_response, check_http_get_response +from swh.web.utils import reverse def test_coverage_view_no_metrics(client, swh_scheduler): diff --git a/swh/web/tests/misc/test_fundraising.py b/swh/web/tests/misc/test_fundraising.py --- a/swh/web/tests/misc/test_fundraising.py +++ b/swh/web/tests/misc/test_fundraising.py @@ -3,9 +3,9 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_contains, assert_not_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse def test_fundraising_banner(client, requests_mock): diff --git a/swh/web/tests/misc/test_iframe.py b/swh/web/tests/misc/test_iframe.py --- a/swh/web/tests/misc/test_iframe.py +++ b/swh/web/tests/misc/test_iframe.py @@ -7,9 +7,9 @@ from swh.model.hashutil import hash_to_bytes from swh.model.swhids import CoreSWHID, ObjectType, QualifiedSWHID -from swh.web.common.utils import reverse from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse def test_content_swhid_iframe(client, content_swhid): diff --git a/swh/web/tests/misc/test_metrics.py b/swh/web/tests/misc/test_metrics.py --- a/swh/web/tests/misc/test_metrics.py +++ b/swh/web/tests/misc/test_metrics.py @@ -10,7 +10,6 @@ from prometheus_client.exposition import CONTENT_TYPE_LATEST import pytest -from swh.web.common.utils import reverse from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_PENDING, @@ -30,7 +29,8 @@ get_savable_visit_types, ) from swh.web.tests.django_asserts import assert_contains -from swh.web.tests.utils import check_http_get_response +from swh.web.tests.helpers import check_http_get_response +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/save_code_now/test_app.py b/swh/web/tests/save_code_now/test_app.py --- a/swh/web/tests/save_code_now/test_app.py +++ b/swh/web/tests/save_code_now/test_app.py @@ -7,10 +7,10 @@ from django.urls import get_resolver -from swh.web.common.utils import reverse from swh.web.save_code_now.urls import urlpatterns from swh.web.tests.django_asserts import assert_not_contains -from swh.web.tests.utils import check_html_get_response +from swh.web.tests.helpers import check_html_get_response +from swh.web.utils import reverse @pytest.mark.django_db diff --git a/swh/web/tests/save_code_now/test_django_command.py b/swh/web/tests/save_code_now/test_django_command.py --- a/swh/web/tests/save_code_now/test_django_command.py +++ b/swh/web/tests/save_code_now/test_django_command.py @@ -11,7 +11,6 @@ from django.core.management import call_command from swh.core.api.classes import stream_results -from swh.web.common.typing import SaveOriginRequestInfo from swh.web.config import get_config from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, @@ -22,6 +21,7 @@ VISIT_STATUS_FULL, VISIT_STATUS_PARTIAL, ) +from swh.web.utils.typing import SaveOriginRequestInfo MODULE_FQDN = "swh.web.save_code_now.management.commands" COMMAND_NAME = "refresh_savecodenow_statuses" diff --git a/swh/web/tests/save_code_now/test_origin_save.py b/swh/web/tests/save_code_now/test_origin_save.py --- a/swh/web/tests/save_code_now/test_origin_save.py +++ b/swh/web/tests/save_code_now/test_origin_save.py @@ -15,12 +15,6 @@ from swh.core.pytest_plugin import get_response_cb from swh.scheduler.utils import create_oneshot_task_dict -from swh.web.common.exc import BadInputExc -from swh.web.common.typing import ( - OriginExistenceCheckInfo, - OriginVisitInfo, - SaveOriginRequestInfo, -) from swh.web.config import get_config from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, @@ -45,6 +39,12 @@ origin_exists, refresh_save_origin_request_statuses, ) +from swh.web.utils.exc import BadInputExc +from swh.web.utils.typing import ( + OriginExistenceCheckInfo, + OriginVisitInfo, + SaveOriginRequestInfo, +) _es_url = "http://esnode1.internal.softwareheritage.org:9200" _es_workers_index_url = "%s/swh_workers-*" % _es_url diff --git a/swh/web/tests/save_code_now/test_origin_save_admin.py b/swh/web/tests/save_code_now/test_origin_save_admin.py --- a/swh/web/tests/save_code_now/test_origin_save_admin.py +++ b/swh/web/tests/save_code_now/test_origin_save_admin.py @@ -7,7 +7,6 @@ import pytest -from swh.web.common.utils import reverse from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_PENDING, @@ -18,7 +17,8 @@ SaveUnauthorizedOrigin, ) from swh.web.save_code_now.origin_save import can_save_origin -from swh.web.tests.utils import check_http_get_response, check_http_post_response +from swh.web.tests.helpers import check_http_get_response, check_http_post_response +from swh.web.utils import reverse _authorized_origin_url = "https://scm.ourproject.org/anonscm/" _unauthorized_origin_url = "https://www.softwareheritage.org/" diff --git a/swh/web/tests/save_code_now/test_origin_save_api.py b/swh/web/tests/save_code_now/test_origin_save_api.py --- a/swh/web/tests/save_code_now/test_origin_save_api.py +++ b/swh/web/tests/save_code_now/test_origin_save_api.py @@ -13,8 +13,6 @@ from swh.web.api.throttling import SwhWebUserRateThrottle from swh.web.auth.utils import API_SAVE_ORIGIN_PERMISSION, SWH_AMBASSADOR_PERMISSION -from swh.web.common.typing import OriginExistenceCheckInfo -from swh.web.common.utils import reverse from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_PENDING, @@ -31,12 +29,14 @@ SaveUnauthorizedOrigin, ) from swh.web.settings.tests import save_origin_rate_post -from swh.web.tests.utils import ( +from swh.web.tests.helpers import ( check_api_get_responses, check_api_post_response, check_api_post_responses, create_django_permission, ) +from swh.web.utils import reverse +from swh.web.utils.typing import OriginExistenceCheckInfo pytestmark = pytest.mark.django_db diff --git a/swh/web/tests/save_code_now/test_origin_save_views.py b/swh/web/tests/save_code_now/test_origin_save_views.py --- a/swh/web/tests/save_code_now/test_origin_save_views.py +++ b/swh/web/tests/save_code_now/test_origin_save_views.py @@ -9,10 +9,10 @@ import pytest from swh.auth.django.utils import oidc_user_from_profile -from swh.web.common.utils import reverse from swh.web.save_code_now.models import SaveOriginRequest from swh.web.save_code_now.origin_save import SAVE_REQUEST_ACCEPTED, SAVE_TASK_SUCCEEDED -from swh.web.tests.utils import check_http_get_response +from swh.web.tests.helpers import check_http_get_response +from swh.web.utils import reverse VISIT_TYPES = ("git", "svn", "hg", "cvs", "bzr") PRIVILEGED_VISIT_TYPES = tuple(list(VISIT_TYPES) + ["archives"]) diff --git a/swh/web/tests/test_templates.py b/swh/web/tests/test_templates.py --- a/swh/web/tests/test_templates.py +++ b/swh/web/tests/test_templates.py @@ -10,10 +10,10 @@ import pytest from swh.web.auth.utils import ADMIN_LIST_DEPOSIT_PERMISSION -from swh.web.common.utils import reverse from swh.web.config import SWH_WEB_SERVER_NAME, SWH_WEB_STAGING_SERVER_NAMES, get_config from swh.web.tests.django_asserts import assert_contains, assert_not_contains -from swh.web.tests.utils import check_http_get_response, create_django_permission +from swh.web.tests.helpers import check_http_get_response, create_django_permission +from swh.web.utils import reverse swh_web_version = get_distribution("swh.web").version @@ -59,7 +59,7 @@ def test_layout_without_oidc_auth_enabled(client, mocker): config = deepcopy(get_config()) config["keycloak"]["server_url"] = "" - mock_get_config = mocker.patch("swh.web.common.utils.get_config") + mock_get_config = mocker.patch("swh.web.utils.get_config") mock_get_config.return_value = config url = reverse("swh-web-homepage") diff --git a/swh/web/tests/common/__init__.py b/swh/web/tests/utils/__init__.py rename from swh/web/tests/common/__init__.py rename to swh/web/tests/utils/__init__.py diff --git a/swh/web/tests/common/test_archive.py b/swh/web/tests/utils/test_archive.py rename from swh/web/tests/common/test_archive.py rename to swh/web/tests/utils/test_archive.py --- a/swh/web/tests/common/test_archive.py +++ b/swh/web/tests/utils/test_archive.py @@ -27,12 +27,12 @@ ) from swh.model.swhids import ObjectType from swh.storage.utils import now -from swh.web.common import archive -from swh.web.common.exc import BadInputExc, NotFoundExc -from swh.web.common.typing import OriginInfo, PagedResult from swh.web.tests.conftest import fossology_missing from swh.web.tests.data import random_content, random_sha1 from swh.web.tests.strategies import new_origin, new_revision, visit_dates +from swh.web.utils import archive +from swh.web.utils.exc import BadInputExc, NotFoundExc +from swh.web.utils.typing import OriginInfo, PagedResult def test_lookup_multiple_hashes_all_present(contents): @@ -975,7 +975,7 @@ ORIGIN = [{"url": origin["url"]}] - mock_archive_search = mocker.patch("swh.web.common.archive.search") + mock_archive_search = mocker.patch("swh.web.utils.archive.search") mock_archive_search.origin_search.return_value = PagedResult( results=ORIGIN, next_page_token=None, diff --git a/swh/web/tests/common/test_converters.py b/swh/web/tests/utils/test_converters.py rename from swh/web/tests/common/test_converters.py rename to swh/web/tests/utils/test_converters.py --- a/swh/web/tests/common/test_converters.py +++ b/swh/web/tests/utils/test_converters.py @@ -15,7 +15,7 @@ RevisionType, TimestampWithTimezone, ) -from swh.web.common import converters +from swh.web.utils import converters def test_fmap(): diff --git a/swh/web/tests/common/test_highlightjs.py b/swh/web/tests/utils/test_highlightjs.py rename from swh/web/tests/common/test_highlightjs.py rename to swh/web/tests/utils/test_highlightjs.py --- a/swh/web/tests/common/test_highlightjs.py +++ b/swh/web/tests/utils/test_highlightjs.py @@ -4,7 +4,7 @@ # See top-level LICENSE file for more information -from swh.web.common import highlightjs +from swh.web.utils import highlightjs def test_get_hljs_language_from_mime_type(): diff --git a/swh/web/tests/common/test_identifiers.py b/swh/web/tests/utils/test_identifiers.py rename from swh/web/tests/common/test_identifiers.py rename to swh/web/tests/utils/test_identifiers.py --- a/swh/web/tests/common/test_identifiers.py +++ b/swh/web/tests/utils/test_identifiers.py @@ -12,8 +12,10 @@ from swh.model.model import Origin from swh.model.swhids import ObjectType, QualifiedSWHID from swh.web.browse.snapshot_context import get_snapshot_context -from swh.web.common.exc import BadInputExc -from swh.web.common.identifiers import ( +from swh.web.tests.data import random_sha1 +from swh.web.utils import reverse +from swh.web.utils.exc import BadInputExc +from swh.web.utils.identifiers import ( gen_swhid, get_swhid, get_swhids_info, @@ -21,9 +23,7 @@ parse_object_type, resolve_swhid, ) -from swh.web.common.typing import SWHObjectInfo -from swh.web.common.utils import reverse -from swh.web.tests.data import random_sha1 +from swh.web.utils.typing import SWHObjectInfo def test_gen_swhid(content): diff --git a/swh/web/tests/common/test_middlewares.py b/swh/web/tests/utils/test_middlewares.py rename from swh/web/tests/common/test_middlewares.py rename to swh/web/tests/utils/test_middlewares.py --- a/swh/web/tests/common/test_middlewares.py +++ b/swh/web/tests/utils/test_middlewares.py @@ -7,11 +7,11 @@ from django.test import modify_settings -from swh.web.common.utils import reverse +from swh.web.utils import reverse @modify_settings( - MIDDLEWARE={"remove": ["swh.web.common.middlewares.ExceptionMiddleware"]} + MIDDLEWARE={"remove": ["swh.web.utils.middlewares.ExceptionMiddleware"]} ) def test_exception_middleware_disabled(client, mocker, snapshot): mock_browse_snapshot_directory = mocker.patch( diff --git a/swh/web/tests/common/test_origin_visits.py b/swh/web/tests/utils/test_origin_visits.py rename from swh/web/tests/common/test_origin_visits.py rename to swh/web/tests/utils/test_origin_visits.py --- a/swh/web/tests/common/test_origin_visits.py +++ b/swh/web/tests/utils/test_origin_visits.py @@ -11,16 +11,16 @@ from swh.model.model import OriginVisit, OriginVisitStatus from swh.storage.utils import now -from swh.web.common.exc import NotFoundExc -from swh.web.common.origin_visits import get_origin_visit, get_origin_visits -from swh.web.common.typing import OriginInfo from swh.web.tests.strategies import new_origin, new_snapshots +from swh.web.utils.exc import NotFoundExc +from swh.web.utils.origin_visits import get_origin_visit, get_origin_visits +from swh.web.utils.typing import OriginInfo @settings(max_examples=1) @given(new_origin(), new_snapshots(3)) def test_get_origin_visits(mocker, archive_data, new_origin, new_snapshots): - from swh.web.common import archive + from swh.web.utils import archive mocker.patch.object(archive, "MAX_LIMIT", 2) @@ -239,7 +239,7 @@ first_visit = origin_visits[0] latest_visit = origin_visits[-1] mock_get_origin_visits = mocker.patch( - "swh.web.common.origin_visits.get_origin_visits" + "swh.web.utils.origin_visits.get_origin_visits" ) mock_get_origin_visits.return_value = origin_visits diff --git a/swh/web/tests/common/test_query.py b/swh/web/tests/utils/test_query.py rename from swh/web/tests/common/test_query.py rename to swh/web/tests/utils/test_query.py --- a/swh/web/tests/common/test_query.py +++ b/swh/web/tests/utils/test_query.py @@ -6,8 +6,8 @@ import pytest from swh.model import hashutil -from swh.web.common import query -from swh.web.common.exc import BadInputExc +from swh.web.utils import query +from swh.web.utils.exc import BadInputExc def test_parse_hash_malformed_query_with_more_than_2_parts(): @@ -71,7 +71,7 @@ def test_parse_hash_with_algorithms_or_throws_bad_query(mocker): - mock_hash = mocker.patch("swh.web.common.query.parse_hash") + mock_hash = mocker.patch("swh.web.utils.query.parse_hash") mock_hash.side_effect = BadInputExc("Error input") with pytest.raises(BadInputExc) as e: @@ -84,7 +84,7 @@ def test_parse_hash_with_algorithms_or_throws_bad_algo(mocker): - mock_hash = mocker.patch("swh.web.common.query.parse_hash") + mock_hash = mocker.patch("swh.web.utils.query.parse_hash") mock_hash.return_value = "sha1", "123" with pytest.raises(BadInputExc) as e: @@ -97,7 +97,7 @@ def test_parse_hash_with_algorithms(mocker): - mock_hash = mocker.patch("swh.web.common.query.parse_hash") + mock_hash = mocker.patch("swh.web.utils.query.parse_hash") mock_hash.return_value = ("sha256", b"123") algo, sha = query.parse_hash_with_algorithms_or_throws( diff --git a/swh/web/tests/common/test_templatetags.py b/swh/web/tests/utils/test_templatetags.py rename from swh/web/tests/common/test_templatetags.py rename to swh/web/tests/utils/test_templatetags.py --- a/swh/web/tests/common/test_templatetags.py +++ b/swh/web/tests/utils/test_templatetags.py @@ -6,7 +6,7 @@ import pytest from swh.web.api.apiresponse import compute_link_header -from swh.web.common.swh_templatetags import ( +from swh.web.utils.swh_templatetags import ( docstring_display, urlize_header_links, urlize_links_and_mails, diff --git a/swh/web/tests/common/test_utils.py b/swh/web/tests/utils/test_utils.py rename from swh/web/tests/common/test_utils.py rename to swh/web/tests/utils/test_utils.py --- a/swh/web/tests/common/test_utils.py +++ b/swh/web/tests/utils/test_utils.py @@ -15,16 +15,30 @@ from django.urls import re_path as url from django.urls.exceptions import NoReverseMatch -from swh.web.common import utils -from swh.web.common.exc import BadInputExc from swh.web.config import SWH_WEB_SERVER_NAME, SWH_WEB_STAGING_SERVER_NAMES, get_config +from swh.web.utils import ( + cache, + django_cache, + format_utc_iso_date, + gen_path_info, + get_deposits_list, + is_swh_web_development, + is_swh_web_production, + is_swh_web_staging, + origin_visit_types, + parse_iso8601_date_to_utc, + reverse, + rst_to_html, + shorten_path, +) +from swh.web.utils.exc import BadInputExc def test_shorten_path_noop(): noops = ["/api/", "/browse/", "/content/symbol/foobar/"] for noop in noops: - assert utils.shorten_path(noop) == noop + assert shorten_path(noop) == noop def test_shorten_path_sha1(): @@ -38,7 +52,7 @@ ] for template in templates: - assert utils.shorten_path(template % sha1) == template % short_sha1 + assert shorten_path(template % sha1) == template % short_sha1 def test_shorten_path_sha256(): @@ -52,7 +66,7 @@ ] for template in templates: - assert utils.shorten_path(template % sha256) == template % short_sha256 + assert shorten_path(template % sha256) == template % short_sha256 @pytest.mark.parametrize( @@ -73,7 +87,7 @@ ], ) def test_parse_iso8601_date_to_utc_ok(input_timestamp, output_date): - assert utils.parse_iso8601_date_to_utc(input_timestamp) == output_date + assert parse_iso8601_date_to_utc(input_timestamp) == output_date @pytest.mark.parametrize( @@ -81,13 +95,12 @@ ) def test_parse_iso8601_date_to_utc_ko(invalid_iso8601_timestamp): with pytest.raises(BadInputExc): - utils.parse_iso8601_date_to_utc(invalid_iso8601_timestamp) + parse_iso8601_date_to_utc(invalid_iso8601_timestamp) def test_format_utc_iso_date(): assert ( - utils.format_utc_iso_date("2017-05-04T13:27:13+02:00") - == "04 May 2017, 11:27:13 UTC" + format_utc_iso_date("2017-05-04T13:27:13+02:00") == "04 May 2017, 11:27:13 UTC" ) @@ -99,11 +112,11 @@ {"name": "swh-environment", "path": "home/user/swh-environment"}, {"name": "swh-web", "path": "home/user/swh-environment/swh-web"}, ] - path_info = utils.gen_path_info(input_path) + path_info = gen_path_info(input_path) assert path_info == expected_result input_path = "home/user/swh-environment/swh-web" - path_info = utils.gen_path_info(input_path) + path_info = gen_path_info(input_path) assert path_info == expected_result @@ -140,7 +153,7 @@ "" ) - assert utils.rst_to_html(rst) == expected_html + assert rst_to_html(rst) == expected_html def sample_test_view(request, string, number): @@ -169,9 +182,7 @@ def test_reverse_url_args_only_ok(): string = "foo" number = 55 - url = utils.reverse( - "sample-test-view", url_args={"string": string, "number": number} - ) + url = reverse("sample-test-view", url_args={"string": string, "number": number}) assert url == f"/sample/test/{string}/view/{number}/" @@ -179,12 +190,12 @@ def test_reverse_url_args_only_ko(): string = "foo" with pytest.raises(NoReverseMatch): - utils.reverse("sample-test-view", url_args={"string": string, "number": string}) + reverse("sample-test-view", url_args={"string": string, "number": string}) @override_settings(ROOT_URLCONF=__name__) def test_reverse_no_url_args(): - url = utils.reverse("sample-test-view-no-url-args") + url = reverse("sample-test-view-no-url-args") assert url == "/sample/test/view/no/url/args/" @@ -192,12 +203,12 @@ def test_reverse_query_params_only(): start = 0 scope = "foo" - url = utils.reverse( + url = reverse( "sample-test-view-no-url-args", query_params={"start": start, "scope": scope} ) assert url == f"/sample/test/view/no/url/args/?scope={scope}&start={start}" - url = utils.reverse( + url = reverse( "sample-test-view-no-url-args", query_params={"start": start, "scope": None} ) assert url == f"/sample/test/view/no/url/args/?start={start}" @@ -206,9 +217,7 @@ @override_settings(ROOT_URLCONF=__name__) def test_reverse_query_params_encode(): libname = "libstc++" - url = utils.reverse( - "sample-test-view-no-url-args", query_params={"libname": libname} - ) + url = reverse("sample-test-view-no-url-args", query_params={"libname": libname}) assert url == f"/sample/test/view/no/url/args/?libname={quote(libname, safe='/;:')}" @@ -218,7 +227,7 @@ number = 55 start = 10 scope = "bar" - url = utils.reverse( + url = reverse( "sample-test-view", url_args={"string": string, "number": number}, query_params={"start": start, "scope": scope}, @@ -228,8 +237,8 @@ @override_settings(ROOT_URLCONF=__name__) def test_reverse_absolute_uri(request_factory): - request = request_factory.get(utils.reverse("sample-test-view-no-url-args")) - url = utils.reverse("sample-test-view-no-url-args", request=request) + request = request_factory.get(reverse("sample-test-view-no-url-args")) + url = reverse("sample-test-view-no-url-args", request=request) assert url == f"http://{request.META['SERVER_NAME']}/sample/test/view/no/url/args/" @@ -286,36 +295,36 @@ }, ) - assert utils.get_deposits_list() == deposits_data["results"] + assert get_deposits_list() == deposits_data["results"] @pytest.mark.parametrize("backend", ["swh-search", "swh-storage"]) def test_origin_visit_types(mocker, backend): if backend != "swh-search": # equivalent to not configuring search in the config - search = mocker.patch("swh.web.common.utils.search") + search = mocker.patch("swh.web.utils.search") search.return_value = None - assert utils.origin_visit_types() == [] + assert origin_visit_types() == [] else: # see swh/web/tests/data.py for origins added for tests - assert utils.origin_visit_types() == ["git", "tar"] + assert origin_visit_types() == ["git", "tar"] @pytest.mark.parametrize("server_name", ["localhost", "127.0.0.1", "testserver"]) def test_is_swh_web_development(request_factory, server_name): request = request_factory.get("/", SERVER_NAME=server_name) - assert utils.is_swh_web_development(request) + assert is_swh_web_development(request) @pytest.mark.parametrize("server_name", SWH_WEB_STAGING_SERVER_NAMES) def test_is_swh_web_staging(request_factory, server_name): request = request_factory.get("/", SERVER_NAME=server_name) - assert utils.is_swh_web_staging(request) + assert is_swh_web_staging(request) def test_is_swh_web_production(request_factory): request = request_factory.get("/", SERVER_NAME=SWH_WEB_SERVER_NAME) - assert utils.is_swh_web_production(request) + assert is_swh_web_production(request) def add(x, y): @@ -326,9 +335,9 @@ """Decorated function should be called once and returned value put in django cache.""" spy_add = mocker.spy(sys.modules[__name__], "add") - spy_cache_set = mocker.spy(utils.cache, "set") + spy_cache_set = mocker.spy(cache, "set") - cached_add = utils.django_cache()(add) + cached_add = django_cache()(add) val = cached_add(1, 2) val2 = cached_add(1, 2) @@ -342,9 +351,9 @@ """Decorated function should be called twice and returned value put in django cache twice.""" spy_add = mocker.spy(sys.modules[__name__], "add") - spy_cache_set = mocker.spy(utils.cache, "set") + spy_cache_set = mocker.spy(cache, "set") - cached_add = utils.django_cache(invalidate_cache_pred=lambda val: val == 3)(add) + cached_add = django_cache(invalidate_cache_pred=lambda val: val == 3)(add) val = cached_add(1, 2) val2 = cached_add(1, 2) @@ -358,9 +367,9 @@ """Decorated function should be called twice, exceptions should be raised and no value put in django cache""" spy_add = mocker.spy(sys.modules[__name__], "add") - spy_cache_set = mocker.spy(utils.cache, "set") + spy_cache_set = mocker.spy(cache, "set") - cached_add = utils.django_cache()(add) + cached_add = django_cache()(add) with pytest.raises(TypeError): cached_add(1, "2") @@ -377,11 +386,11 @@ raised, specified fallback value should be returned and no value put in django cache""" spy_add = mocker.spy(sys.modules[__name__], "add") - spy_cache_set = mocker.spy(utils.cache, "set") + spy_cache_set = mocker.spy(cache, "set") - cached_add = utils.django_cache( - catch_exception=True, exception_return_value=math.nan - )(add) + cached_add = django_cache(catch_exception=True, exception_return_value=math.nan)( + add + ) val = cached_add(1, "2") val2 = cached_add(1, "2") diff --git a/swh/web/tests/views.py b/swh/web/tests/views.py --- a/swh/web/tests/views.py +++ b/swh/web/tests/views.py @@ -16,8 +16,8 @@ from swh.model.from_disk import DiskBackedContent from swh.model.hashutil import hash_to_hex from swh.model.model import Content -from swh.web.common.highlightjs import get_hljs_language_from_filename from swh.web.tests.data import get_tests_data +from swh.web.utils.highlightjs import get_hljs_language_from_filename _content_code_data_exts: Dict[str, Dict[str, str]] = {} _content_code_data_filenames: Dict[str, Dict[str, str]] = {} diff --git a/swh/web/urls.py b/swh/web/urls.py --- a/swh/web/urls.py +++ b/swh/web/urls.py @@ -15,14 +15,9 @@ from django.views.generic.base import RedirectView from swh.web.browse.identifiers import swhid_browse -from swh.web.common.exc import ( - swh_handle400, - swh_handle403, - swh_handle404, - swh_handle500, -) -from swh.web.common.utils import origin_visit_types from swh.web.config import get_config +from swh.web.utils import origin_visit_types +from swh.web.utils.exc import swh_handle400, swh_handle403, swh_handle404, swh_handle500 swh_web_config = get_config() diff --git a/swh/web/utils/__init__.py b/swh/web/utils/__init__.py --- a/swh/web/utils/__init__.py +++ b/swh/web/utils/__init__.py @@ -0,0 +1,523 @@ +# Copyright (C) 2017-2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from datetime import datetime, timezone +import functools +import os +import re +from typing import Any, Callable, Dict, List, Mapping, Optional +import urllib.parse + +from bs4 import BeautifulSoup +from docutils.core import publish_parts +import docutils.parsers.rst +import docutils.utils +from docutils.writers.html5_polyglot import HTMLTranslator, Writer +from iso8601 import ParseError, parse_date +from pkg_resources import get_distribution +from prometheus_client.registry import CollectorRegistry +import requests +from requests.auth import HTTPBasicAuth + +from django.conf import settings +from django.core.cache import cache +from django.core.cache.backends.base import DEFAULT_TIMEOUT +from django.http import HttpRequest, QueryDict +from django.shortcuts import redirect +from django.urls import resolve +from django.urls import reverse as django_reverse + +from swh.web.auth.utils import ( + ADD_FORGE_MODERATOR_PERMISSION, + ADMIN_LIST_DEPOSIT_PERMISSION, + MAILMAP_ADMIN_PERMISSION, +) +from swh.web.config import SWH_WEB_SERVER_NAME, get_config, search +from swh.web.utils.exc import BadInputExc, sentry_capture_exception + +SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) + +SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}" + + +swh_object_icons = { + "alias": "mdi mdi-star", + "branch": "mdi mdi-source-branch", + "branches": "mdi mdi-source-branch", + "content": "mdi mdi-file-document", + "cnt": "mdi mdi-file-document", + "directory": "mdi mdi-folder", + "dir": "mdi mdi-folder", + "origin": "mdi mdi-source-repository", + "ori": "mdi mdi-source-repository", + "person": "mdi mdi-account", + "revisions history": "mdi mdi-history", + "release": "mdi mdi-tag", + "rel": "mdi mdi-tag", + "releases": "mdi mdi-tag", + "revision": "mdi mdi-rotate-90 mdi-source-commit", + "rev": "mdi mdi-rotate-90 mdi-source-commit", + "snapshot": "mdi mdi-camera", + "snp": "mdi mdi-camera", + "visits": "mdi mdi-calendar-month", +} + + +def reverse( + viewname: str, + url_args: Optional[Dict[str, Any]] = None, + query_params: Optional[Mapping[str, Optional[str]]] = None, + current_app: Optional[str] = None, + urlconf: Optional[str] = None, + request: Optional[HttpRequest] = None, +) -> str: + """An override of django reverse function supporting query parameters. + + Args: + viewname: the name of the django view from which to compute a url + url_args: dictionary of url arguments indexed by their names + query_params: dictionary of query parameters to append to the + reversed url + current_app: the name of the django app tighten to the view + urlconf: url configuration module + request: build an absolute URI if provided + + Returns: + str: the url of the requested view with processed arguments and + query parameters + """ + + if url_args: + url_args = {k: v for k, v in url_args.items() if v is not None} + + url = django_reverse( + viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app + ) + + params: Dict[str, str] = {} + if query_params: + params = {k: v for k, v in query_params.items() if v is not None} + + if params: + query_dict = QueryDict("", mutable=True) + query_dict.update(dict(sorted(params.items()))) + url += "?" + query_dict.urlencode(safe="/;:") + + if request is not None: + url = request.build_absolute_uri(url) + + return url + + +def datetime_to_utc(date): + """Returns datetime in UTC without timezone info + + Args: + date (datetime.datetime): input datetime with timezone info + + Returns: + datetime.datetime: datetime in UTC without timezone info + """ + if date.tzinfo and date.tzinfo != timezone.utc: + return date.astimezone(tz=timezone.utc) + else: + return date + + +def parse_iso8601_date_to_utc(iso_date: str) -> datetime: + """Given an ISO 8601 datetime string, parse the result as UTC datetime. + + Returns: + a timezone-aware datetime representing the parsed date + + Raises: + swh.web.utils.exc.BadInputExc: provided date does not respect ISO 8601 format + + Samples: + - 2016-01-12 + - 2016-01-12T09:19:12+0100 + - 2007-01-14T20:34:22Z + + """ + try: + date = parse_date(iso_date) + return datetime_to_utc(date) + except ParseError as e: + raise BadInputExc(e) + + +def shorten_path(path): + """Shorten the given path: for each hash present, only return the first + 8 characters followed by an ellipsis""" + + sha256_re = r"([0-9a-f]{8})[0-9a-z]{56}" + sha1_re = r"([0-9a-f]{8})[0-9a-f]{32}" + + ret = re.sub(sha256_re, r"\1...", path) + return re.sub(sha1_re, r"\1...", ret) + + +def format_utc_iso_date(iso_date, fmt="%d %B %Y, %H:%M:%S UTC"): + """Turns a string representation of an ISO 8601 datetime string + to UTC and format it into a more human readable one. + + For instance, from the following input + string: '2017-05-04T13:27:13+02:00' the following one + is returned: '04 May 2017, 11:27 UTC'. + Custom format string may also be provided + as parameter + + Args: + iso_date (str): a string representation of an ISO 8601 date + fmt (str): optional date formatting string + + Returns: + str: a formatted string representation of the input iso date + """ + if not iso_date: + return iso_date + date = parse_iso8601_date_to_utc(iso_date) + return date.strftime(fmt) + + +def gen_path_info(path): + """Function to generate path data navigation for use + with a breadcrumb in the swh web ui. + + For instance, from a path /folder1/folder2/folder3, + it returns the following list:: + + [{'name': 'folder1', 'path': 'folder1'}, + {'name': 'folder2', 'path': 'folder1/folder2'}, + {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] + + Args: + path: a filesystem path + + Returns: + list: a list of path data for navigation as illustrated above. + + """ + path_info = [] + if path: + sub_paths = path.strip("/").split("/") + path_from_root = "" + for p in sub_paths: + path_from_root += "/" + p + path_info.append({"name": p, "path": path_from_root.strip("/")}) + return path_info + + +def parse_rst(text, report_level=2): + """ + Parse a reStructuredText string with docutils. + + Args: + text (str): string with reStructuredText markups in it + report_level (int): level of docutils report messages to print + (1 info 2 warning 3 error 4 severe 5 none) + + Returns: + docutils.nodes.document: a parsed docutils document + """ + parser = docutils.parsers.rst.Parser() + components = (docutils.parsers.rst.Parser,) + settings = docutils.frontend.OptionParser( + components=components + ).get_default_values() + settings.report_level = report_level + document = docutils.utils.new_document("rst-doc", settings=settings) + parser.parse(text, document) + return document + + +def get_client_ip(request): + """ + Return the client IP address from an incoming HTTP request. + + Args: + request (django.http.HttpRequest): the incoming HTTP request + + Returns: + str: The client IP address + """ + x_forwarded_for = request.META.get("HTTP_X_FORWARDED_FOR") + if x_forwarded_for: + ip = x_forwarded_for.split(",")[0] + else: + ip = request.META.get("REMOTE_ADDR") + return ip + + +def is_swh_web_development(request: HttpRequest) -> bool: + """Indicate if we are running a development version of swh-web.""" + site_base_url = request.build_absolute_uri("/") + return any( + host in site_base_url for host in ("localhost", "127.0.0.1", "testserver") + ) + + +def is_swh_web_staging(request: HttpRequest) -> bool: + """Indicate if we are running a staging version of swh-web.""" + config = get_config() + site_base_url = request.build_absolute_uri("/") + return any( + server_name in site_base_url for server_name in config["staging_server_names"] + ) + + +def is_swh_web_production(request: HttpRequest) -> bool: + """Indicate if we are running the public production version of swh-web.""" + return SWH_WEB_SERVER_NAME in request.build_absolute_uri("/") + + +browsers_supported_image_mimes = set( + [ + "image/gif", + "image/png", + "image/jpeg", + "image/bmp", + "image/webp", + "image/svg", + "image/svg+xml", + ] +) + + +def context_processor(request): + """ + Django context processor used to inject variables + in all swh-web templates. + """ + config = get_config() + if ( + hasattr(request, "user") + and request.user.is_authenticated + and not hasattr(request.user, "backend") + ): + # To avoid django.template.base.VariableDoesNotExist errors + # when rendering templates when standard Django user is logged in. + request.user.backend = "django.contrib.auth.backends.ModelBackend" + + return { + "swh_object_icons": swh_object_icons, + "available_languages": None, + "swh_client_config": config["client_config"], + "oidc_enabled": bool(config["keycloak"]["server_url"]), + "browsers_supported_image_mimes": browsers_supported_image_mimes, + "keycloak": config["keycloak"], + "site_base_url": request.build_absolute_uri("/"), + "DJANGO_SETTINGS_MODULE": os.environ["DJANGO_SETTINGS_MODULE"], + "status": config["status"], + "swh_web_dev": is_swh_web_development(request), + "swh_web_staging": is_swh_web_staging(request), + "swh_web_prod": is_swh_web_production(request), + "swh_web_version": get_distribution("swh.web").version, + "iframe_mode": False, + "ADMIN_LIST_DEPOSIT_PERMISSION": ADMIN_LIST_DEPOSIT_PERMISSION, + "ADD_FORGE_MODERATOR_PERMISSION": ADD_FORGE_MODERATOR_PERMISSION, + "MAILMAP_ADMIN_PERMISSION": MAILMAP_ADMIN_PERMISSION, + "lang": "en", + "sidebar_state": request.COOKIES.get("sidebar-state", "expanded"), + "SWH_DJANGO_APPS": settings.SWH_DJANGO_APPS, + } + + +def resolve_branch_alias( + snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] +) -> Optional[Dict[str, Any]]: + """ + Resolve branch alias in snapshot content. + + Args: + snapshot: a full snapshot content + branch: a branch alias contained in the snapshot + Returns: + The real snapshot branch that got aliased. + """ + while branch and branch["target_type"] == "alias": + if branch["target"] in snapshot["branches"]: + branch = snapshot["branches"][branch["target"]] + else: + from swh.web.utils import archive + + snp = archive.lookup_snapshot( + snapshot["id"], branches_from=branch["target"], branches_count=1 + ) + if snp and branch["target"] in snp["branches"]: + branch = snp["branches"][branch["target"]] + else: + branch = None + return branch + + +class _NoHeaderHTMLTranslator(HTMLTranslator): + """ + Docutils translator subclass to customize the generation of HTML + from reST-formatted docstrings + """ + + def __init__(self, document): + super().__init__(document) + self.body_prefix = [] + self.body_suffix = [] + + +_HTML_WRITER = Writer() +_HTML_WRITER.translator_class = _NoHeaderHTMLTranslator + + +def rst_to_html(rst: str) -> str: + """ + Convert reStructuredText document into HTML. + + Args: + rst: A string containing a reStructuredText document + + Returns: + Body content of the produced HTML conversion. + + """ + settings = { + "initial_header_level": 2, + "halt_level": 4, + "traceback": True, + "file_insertion_enabled": False, + "raw_enabled": False, + } + pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) + return f'
{pp["html_body"]}
' + + +def prettify_html(html: str) -> str: + """ + Prettify an HTML document. + + Args: + html: Input HTML document + + Returns: + The prettified HTML document + """ + return BeautifulSoup(html, "lxml").prettify() + + +def django_cache( + timeout: int = DEFAULT_TIMEOUT, + catch_exception: bool = False, + exception_return_value: Any = None, + invalidate_cache_pred: Callable[[Any], bool] = lambda val: False, +): + """Decorator to put the result of a function call in Django cache, + subsequent calls will directly return the cached value. + + Args: + timeout: The number of seconds value will be hold in cache + catch_exception: If :const:`True`, any thrown exception by + the decorated function will be caught and not reraised + exception_return_value: The value to return if previous + parameter is set to :const:`True` + invalidate_cache_pred: A predicate function enabling to + invalidate the cache under certain conditions, decorated + function will then be called again + + Returns: + The returned value of the decorated function for the specified + parameters + + """ + + def inner(func): + @functools.wraps(func) + def wrapper(*args, **kwargs): + func_args = args + (0,) + tuple(sorted(kwargs.items())) + cache_key = str(hash((func.__module__, func.__name__) + func_args)) + ret = cache.get(cache_key) + if ret is None or invalidate_cache_pred(ret): + try: + ret = func(*args, **kwargs) + except Exception as exc: + if catch_exception: + sentry_capture_exception(exc) + return exception_return_value + else: + raise + else: + cache.set(cache_key, ret, timeout=timeout) + return ret + + return wrapper + + return inner + + +def _deposits_list_url( + deposits_list_base_url: str, page_size: int, username: Optional[str] +) -> str: + params = {"page_size": str(page_size)} + if username is not None: + params["username"] = username + return f"{deposits_list_base_url}?{urllib.parse.urlencode(params)}" + + +def get_deposits_list(username: Optional[str] = None) -> List[Dict[str, Any]]: + """Return the list of software deposits using swh-deposit API""" + config = get_config()["deposit"] + private_api_url = config["private_api_url"].rstrip("/") + "/" + deposits_list_base_url = private_api_url + "deposits" + deposits_list_auth = HTTPBasicAuth( + config["private_api_user"], config["private_api_password"] + ) + + deposits_list_url = _deposits_list_url( + deposits_list_base_url, page_size=1, username=username + ) + + nb_deposits = requests.get( + deposits_list_url, auth=deposits_list_auth, timeout=30 + ).json()["count"] + + @django_cache(invalidate_cache_pred=lambda data: data["count"] != nb_deposits) + def _get_deposits_data(): + deposits_list_url = _deposits_list_url( + deposits_list_base_url, page_size=nb_deposits, username=username + ) + return requests.get( + deposits_list_url, + auth=deposits_list_auth, + timeout=30, + ).json() + + deposits_data = _get_deposits_data() + + return deposits_data["results"] + + +_origin_visit_types_cache_timeout = 24 * 60 * 60 # 24 hours + + +@django_cache( + timeout=_origin_visit_types_cache_timeout, + catch_exception=True, + exception_return_value=[], +) +def origin_visit_types() -> List[str]: + """Return the exhaustive list of visit types for origins + ingested into the archive. + """ + return sorted(search().visit_types_count().keys()) + + +def redirect_to_new_route(request, new_route, permanent=True): + """Redirect a request to another route with url args and query parameters + eg: /origin//log?path=test can be redirected as + /log?url=&path=test. This can be used to deprecate routes + """ + request_path = resolve(request.path_info) + args = {**request_path.kwargs, **request.GET.dict()} + return redirect( + reverse(new_route, query_params=args), + permanent=permanent, + ) diff --git a/swh/web/common/archive.py b/swh/web/utils/archive.py rename from swh/web/common/archive.py rename to swh/web/utils/archive.py --- a/swh/web/common/archive.py +++ b/swh/web/utils/archive.py @@ -20,9 +20,9 @@ from swh.storage.interface import OriginVisitWithStatuses from swh.vault.exc import NotFoundExc as VaultNotFoundExc from swh.web import config -from swh.web.common import converters, query -from swh.web.common.exc import NotFoundExc -from swh.web.common.typing import ( +from swh.web.utils import converters, query +from swh.web.utils.exc import NotFoundExc +from swh.web.utils.typing import ( OriginInfo, OriginMetadataInfo, OriginVisitInfo, @@ -604,7 +604,7 @@ else: raise TypeError('"origin" must be an int or a string.') - from swh.web.common.origin_visits import get_origin_visit + from swh.web.utils.origin_visits import get_origin_visit visit = get_origin_visit(origin, visit_ts=timestamp) branch = _get_snapshot_branch(visit["snapshot"], branch_name) @@ -659,7 +659,7 @@ Raises: ValueError: if the identifier provided is not of sha1 nature. - swh.web.common.exc.NotFoundExc: if there is no revision with the + swh.web.utils.exc.NotFoundExc: if there is no revision with the provided sha1_git. """ @@ -682,7 +682,7 @@ list: Revision log as list of revision dicts Raises: - swh.web.common.exc.NotFoundExc: if no revision corresponds to the + swh.web.utils.exc.NotFoundExc: if no revision corresponds to the criterion """ @@ -1364,7 +1364,7 @@ dictionary for the directory object type. Raises: - swh.web.common.exc.NotFoundExc: if the object could not be found in + swh.web.utils.exc.NotFoundExc: if the object could not be found in the archive BadInputExc: if the object identifier is invalid """ diff --git a/swh/web/common/converters.py b/swh/web/utils/converters.py rename from swh/web/common/converters.py rename to swh/web/utils/converters.py --- a/swh/web/common/converters.py +++ b/swh/web/utils/converters.py @@ -19,7 +19,7 @@ ) from swh.model.swhids import ObjectType from swh.storage.interface import PartialBranches -from swh.web.common.typing import OriginInfo, OriginVisitInfo +from swh.web.utils.typing import OriginInfo, OriginVisitInfo def _group_checksums(data): @@ -224,7 +224,7 @@ class SWHDjangoJSONEncoder(DjangoJSONEncoder): """Wrapper around DjangoJSONEncoder to serialize SWH-specific types - found in :class:`swh.web.common.typing.SWHObjectInfo`.""" + found in :class:`swh.web.utils.typing.SWHObjectInfo`.""" def default(self, o): if isinstance(o, ObjectType): diff --git a/swh/web/common/exc.py b/swh/web/utils/exc.py rename from swh/web/common/exc.py rename to swh/web/utils/exc.py diff --git a/swh/web/common/highlightjs.py b/swh/web/utils/highlightjs.py rename from swh/web/common/highlightjs.py rename to swh/web/utils/highlightjs.py --- a/swh/web/common/highlightjs.py +++ b/swh/web/utils/highlightjs.py @@ -11,7 +11,7 @@ from django.contrib.staticfiles.finders import find -from swh.web.common.exc import sentry_capture_exception +from swh.web.utils.exc import sentry_capture_exception @functools.lru_cache() diff --git a/swh/web/common/identifiers.py b/swh/web/utils/identifiers.py rename from swh/web/common/identifiers.py rename to swh/web/utils/identifiers.py --- a/swh/web/common/identifiers.py +++ b/swh/web/utils/identifiers.py @@ -11,15 +11,9 @@ from swh.model.exceptions import ValidationError from swh.model.hashutil import hash_to_bytes, hash_to_hex from swh.model.swhids import ObjectType, QualifiedSWHID -from swh.web.common import archive -from swh.web.common.exc import BadInputExc -from swh.web.common.typing import ( - SnapshotContext, - SWHIDContext, - SWHIDInfo, - SWHObjectInfo, -) -from swh.web.common.utils import reverse +from swh.web.utils import archive, reverse +from swh.web.utils.exc import BadInputExc +from swh.web.utils.typing import SnapshotContext, SWHIDContext, SWHIDInfo, SWHObjectInfo def parse_object_type(object_type: str) -> ObjectType: diff --git a/swh/web/common/middlewares.py b/swh/web/utils/middlewares.py rename from swh/web/common/middlewares.py rename to swh/web/utils/middlewares.py --- a/swh/web/common/middlewares.py +++ b/swh/web/utils/middlewares.py @@ -6,8 +6,8 @@ from htmlmin import minify -from swh.web.common.exc import handle_view_exception, sentry_capture_exception -from swh.web.common.utils import prettify_html +from swh.web.utils import prettify_html +from swh.web.utils.exc import handle_view_exception, sentry_capture_exception class HtmlPrettifyMiddleware(object): diff --git a/swh/web/common/origin_visits.py b/swh/web/utils/origin_visits.py rename from swh/web/common/origin_visits.py rename to swh/web/utils/origin_visits.py --- a/swh/web/common/origin_visits.py +++ b/swh/web/utils/origin_visits.py @@ -7,10 +7,9 @@ from django.core.cache import cache -from swh.web.common import archive -from swh.web.common.exc import NotFoundExc -from swh.web.common.typing import OriginInfo, OriginVisitInfo -from swh.web.common.utils import parse_iso8601_date_to_utc +from swh.web.utils import archive, parse_iso8601_date_to_utc +from swh.web.utils.exc import NotFoundExc +from swh.web.utils.typing import OriginInfo, OriginVisitInfo def get_origin_visits( @@ -32,10 +31,10 @@ A list of dict describing the origin visits Raises: - swh.web.common.exc.NotFoundExc: if the origin is not found + swh.web.utils.exc.NotFoundExc: if the origin is not found """ - from swh.web.common import archive + from swh.web.utils import archive origin_url = archive.lookup_origin( origin_info, lookup_similar_urls=lookup_similar_urls @@ -113,7 +112,7 @@ A dict containing the visit info. Raises: - swh.web.common.exc.NotFoundExc: if no visit can be found + swh.web.utils.exc.NotFoundExc: if no visit can be found """ # returns the latest full visit with a valid snapshot visit = archive.lookup_origin_visit_latest( diff --git a/swh/web/common/query.py b/swh/web/utils/query.py rename from swh/web/common/query.py rename to swh/web/utils/query.py --- a/swh/web/common/query.py +++ b/swh/web/utils/query.py @@ -7,7 +7,7 @@ import re from swh.model.hashutil import ALGORITHMS, hash_to_bytes -from swh.web.common.exc import BadInputExc +from swh.web.utils.exc import BadInputExc SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) SHA1_RE = re.compile(r"^[0-9a-f]{40}$", re.IGNORECASE) diff --git a/swh/web/common/swh_templatetags.py b/swh/web/utils/swh_templatetags.py rename from swh/web/common/swh_templatetags.py rename to swh/web/utils/swh_templatetags.py --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/utils/swh_templatetags.py @@ -9,9 +9,9 @@ from django import template from django.utils.safestring import mark_safe -from swh.web.common.converters import SWHDjangoJSONEncoder -from swh.web.common.utils import rst_to_html from swh.web.save_code_now.origin_save import get_savable_visit_types +from swh.web.utils import rst_to_html +from swh.web.utils.converters import SWHDjangoJSONEncoder register = template.Library() diff --git a/swh/web/common/typing.py b/swh/web/utils/typing.py rename from swh/web/common/typing.py rename to swh/web/utils/typing.py diff --git a/swh/web/common/urlsindex.py b/swh/web/utils/urlsindex.py rename from swh/web/common/urlsindex.py rename to swh/web/utils/urlsindex.py