diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py index 58f93157..d3294407 100644 --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -1,125 +1,125 @@ -# Copyright (C) 2017-2019 The Software Heritage developers +# Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from typing import Dict, List, Optional from django.http.response import HttpResponseBase from rest_framework.decorators import api_view from swh.web.api import throttling from swh.web.api.apiresponse import make_api_response from swh.web.common.urlsindex import UrlsIndex class APIUrls(UrlsIndex): """ Class to manage API documentation URLs. - Indexes all routes documented using apidoc's decorators. - Tracks endpoint/request processing method relationships for use in generating related urls in API documentation """ - _apidoc_routes = {} # type: Dict[str, Dict[str, str]] + _apidoc_routes: Dict[str, Dict[str, str]] = {} scope = "api" @classmethod def get_app_endpoints(cls) -> Dict[str, Dict[str, str]]: return cls._apidoc_routes @classmethod def add_doc_route( cls, route: str, docstring: str, noargs: bool = False, api_version: str = "1", **kwargs, ) -> None: """ Add a route to the self-documenting API reference """ route_name = route[1:-1].replace("/", "-") if not noargs: route_name = "%s-doc" % route_name route_view_name = "api-%s-%s" % (api_version, route_name) if route not in cls._apidoc_routes: d = { "docstring": docstring, "route": "/api/%s%s" % (api_version, route), "route_view_name": route_view_name, } for k, v in kwargs.items(): d[k] = v cls._apidoc_routes[route] = d def api_route( url_pattern: str, view_name: Optional[str] = None, methods: List[str] = ["GET", "HEAD", "OPTIONS"], throttle_scope: str = "swh_api", api_version: str = "1", checksum_args: Optional[List[str]] = None, never_cache: bool = False, ): """ Decorator to ease the registration of an API endpoint using the Django REST Framework. Args: url_pattern: the url pattern used by DRF to identify the API route view_name: the name of the API view associated to the route used to reverse the url methods: array of HTTP methods supported by the API route throttle_scope: Named scope for rate limiting api_version: web API version checksum_args: list of view argument names holding checksum values never_cache: define if api response must be cached """ url_pattern = "^" + api_version + url_pattern + "$" def decorator(f): # create a DRF view from the wrapped function @api_view(methods) @throttling.throttle_scope(throttle_scope) @functools.wraps(f) def api_view_f(request, **kwargs): # never_cache will be handled in apiresponse module request.never_cache = never_cache response = f(request, **kwargs) doc_data = None # check if response has been forwarded by api_doc decorator if isinstance(response, dict) and "doc_data" in response: doc_data = response["doc_data"] response = response["data"] # check if HTTP response needs to be created if not isinstance(response, HttpResponseBase): api_response = make_api_response( request, data=response, doc_data=doc_data ) else: api_response = response return api_response # small hacks for correctly generating API endpoints index doc api_view_f.__name__ = f.__name__ api_view_f.http_method_names = methods # register the route and its view in the endpoints index APIUrls.add_url_pattern(url_pattern, api_view_f, view_name) if checksum_args: APIUrls.add_redirect_for_checksum_args( view_name, [url_pattern], checksum_args ) return f return decorator diff --git a/swh/web/common/highlightjs.py b/swh/web/common/highlightjs.py index 362bfd92..ee52bcbc 100644 --- a/swh/web/common/highlightjs.py +++ b/swh/web/common/highlightjs.py @@ -1,184 +1,184 @@ # Copyright (C) 2017-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools import json from typing import Dict from pygments.lexers import get_all_lexers, get_lexer_for_filename from django.contrib.staticfiles.finders import find from swh.web.common.exc import sentry_capture_exception @functools.lru_cache() def _hljs_languages_data(): with open(str(find("json/highlightjs-languages.json")), "r") as hljs_languages_file: return json.load(hljs_languages_file) # set of languages ids that can be highlighted by highlight.js library @functools.lru_cache() def _hljs_languages(): return set(_hljs_languages_data()["languages"]) # languages aliases defined in highlight.js @functools.lru_cache() def _hljs_languages_aliases(): language_aliases = _hljs_languages_data()["languages_aliases"] language_aliases.pop("robots.txt", None) return { **language_aliases, "ml": "ocaml", "bsl": "1c", "ep": "mojolicious", "lc": "livecode", "p": "parser3", "pde": "processing", "rsc": "routeros", "s": "armasm", "sl": "rsl", "4dm": "4d", "kaos": "chaos", "dfy": "dafny", "ejs": "eta", "nev": "never", "m": "octave", "shader": "hlsl", "fx": "hlsl", "prg": "xsharp", "xs": "xsharp", } # dictionary mapping pygment lexers to hljs languages -_pygments_lexer_to_hljs_language = {} # type: Dict[str, str] +_pygments_lexer_to_hljs_language: Dict[str, str] = {} # dictionary mapping mime types to hljs languages _mime_type_to_hljs_language = { "text/x-c": "c", "text/x-c++": "cpp", "text/x-msdos-batch": "dos", "text/x-lisp": "lisp", "text/x-shellscript": "bash", } # dictionary mapping filenames to hljs languages _filename_to_hljs_language = { "cmakelists.txt": "cmake", ".htaccess": "apache", "httpd.conf": "apache", "access.log": "accesslog", "nginx.log": "accesslog", "resolv.conf": "dns", "dockerfile": "docker", "nginx.conf": "nginx", "pf.conf": "pf", "robots.txt": "robots-txt", } # function to fill the above dictionaries def _init_pygments_to_hljs_map(): if len(_pygments_lexer_to_hljs_language) == 0: hljs_languages = _hljs_languages() hljs_languages_aliases = _hljs_languages_aliases() for lexer in get_all_lexers(): lexer_name = lexer[0] lang_aliases = lexer[1] lang_mime_types = lexer[3] lang = None for lang_alias in lang_aliases: if lang_alias in hljs_languages: lang = lang_alias _pygments_lexer_to_hljs_language[lexer_name] = lang_alias break if lang_alias in hljs_languages_aliases: lang = hljs_languages_aliases[lang_alias] _pygments_lexer_to_hljs_language[lexer_name] = lang_alias break if lang: for lang_mime_type in lang_mime_types: if lang_mime_type not in _mime_type_to_hljs_language: _mime_type_to_hljs_language[lang_mime_type] = lang def get_hljs_language_from_filename(filename): """Function that tries to associate a language supported by highlight.js from a filename. Args: filename: input filename Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if filename: filename_lower = filename.lower() if filename_lower in _filename_to_hljs_language: return _filename_to_hljs_language[filename_lower] if filename_lower in _hljs_languages(): return filename_lower exts = filename_lower.split(".") # check if file extension matches an hljs language # also handle .ext.in cases for ext in reversed(exts[-2:]): if ext in _hljs_languages(): return ext if ext in _hljs_languages_aliases(): return _hljs_languages_aliases()[ext] # otherwise use Pygments language database lexer = None # try to find a Pygment lexer try: lexer = get_lexer_for_filename(filename) except Exception as exc: sentry_capture_exception(exc) # if there is a correspondence between the lexer and an hljs # language, return it if lexer and lexer.name in _pygments_lexer_to_hljs_language: return _pygments_lexer_to_hljs_language[lexer.name] # otherwise, try to find a match between the file extensions # associated to the lexer and the hljs language aliases if lexer: exts = [ext.replace("*.", "") for ext in lexer.filenames] for ext in exts: if ext in _hljs_languages_aliases(): return _hljs_languages_aliases()[ext] return None def get_hljs_language_from_mime_type(mime_type): """Function that tries to associate a language supported by highlight.js from a mime type. Args: mime_type: input mime type Returns: highlight.js language id or None if no correspondence has been found """ _init_pygments_to_hljs_map() if mime_type and mime_type in _mime_type_to_hljs_language: return _mime_type_to_hljs_language[mime_type] return None @functools.lru_cache() def get_supported_languages(): """ Return the list of programming languages that can be highlighted using the highlight.js library. Returns: List[str]: the list of supported languages """ return sorted(list(_hljs_languages())) diff --git a/swh/web/tests/views.py b/swh/web/tests/views.py index 3f90d7d5..f175a841 100644 --- a/swh/web/tests/views.py +++ b/swh/web/tests/views.py @@ -1,171 +1,171 @@ -# Copyright (C) 2018-2020 The Software Heritage developers +# Copyright (C) 2018-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Implement some special endpoints used to provide input tests data # when executing end to end tests with cypress import os from typing import Dict from rest_framework.decorators import api_view from rest_framework.response import Response from swh.model import from_disk from swh.model.from_disk import DiskBackedContent from swh.model.hashutil import hash_to_hex from swh.model.model import Content from swh.web.common.highlightjs import get_hljs_language_from_filename from swh.web.tests.data import get_tests_data -_content_code_data_exts = {} # type: Dict[str, Dict[str, str]] -_content_code_data_filenames = {} # type: Dict[str, Dict[str, str]] -_content_other_data_exts = {} # type: Dict[str, Dict[str, str]] +_content_code_data_exts: Dict[str, Dict[str, str]] = {} +_content_code_data_filenames: Dict[str, Dict[str, str]] = {} +_content_other_data_exts: Dict[str, Dict[str, str]] = {} def _init_content_tests_data(data_path, data_dict, ext_key): """ Helper function to read the content of a directory, store it into a test archive and add some files metadata (sha1 and/or expected programming language) in a dict. Args: data_path (str): path to a directory relative to the tests folder of swh-web data_dict (dict): the dict that will store files metadata ext_key (bool): whether to use file extensions or filenames as dict keys """ test_contents_dir = os.path.join(os.path.dirname(__file__), data_path).encode( "utf-8" ) directory = from_disk.Directory.from_disk(path=test_contents_dir) contents = [] for name, obj_ in directory.items(): obj = obj_.to_model() if obj.object_type in [Content.object_type, DiskBackedContent.object_type]: c = obj.with_data().to_dict() c["status"] = "visible" sha1 = hash_to_hex(c["sha1"]) if ext_key: key = name.decode("utf-8").split(".")[-1] filename = "test." + key else: filename = name.decode("utf-8").split("/")[-1] key = filename language = get_hljs_language_from_filename(filename) data_dict[key] = {"sha1": sha1, "language": language} contents.append(Content.from_dict(c)) storage = get_tests_data()["storage"] storage.content_add(contents) def _init_content_code_data_exts(): """ Fill a global dictionary which maps source file extension to a code content example. """ global _content_code_data_exts if not _content_code_data_exts: _init_content_tests_data( "resources/contents/code/extensions", _content_code_data_exts, True ) def _init_content_other_data_exts(): """ Fill a global dictionary which maps a file extension to a content example. """ global _content_other_data_exts if not _content_other_data_exts: _init_content_tests_data( "resources/contents/other/extensions", _content_other_data_exts, True ) def _init_content_code_data_filenames(): """ Fill a global dictionary which maps a filename to a content example. """ global _content_code_data_filenames if not _content_code_data_filenames: _init_content_tests_data( "resources/contents/code/filenames", _content_code_data_filenames, False ) @api_view(["GET"]) def get_content_code_data_all_exts(request): """ Endpoint implementation returning a list of all source file extensions to test for highlighting using cypress. """ _init_content_code_data_exts() return Response( sorted(_content_code_data_exts.keys()), status=200, content_type="application/json", ) @api_view(["GET"]) def get_content_code_data_by_ext(request, ext): """ Endpoint implementation returning metadata of a code content example based on the source file extension. """ data = None status = 404 _init_content_code_data_exts() if ext in _content_code_data_exts: data = _content_code_data_exts[ext] status = 200 return Response(data, status=status, content_type="application/json") @api_view(["GET"]) def get_content_other_data_by_ext(request, ext): """ Endpoint implementation returning metadata of a content example based on the file extension. """ _init_content_other_data_exts() data = None status = 404 if ext in _content_other_data_exts: data = _content_other_data_exts[ext] status = 200 return Response(data, status=status, content_type="application/json") @api_view(["GET"]) def get_content_code_data_all_filenames(request): """ Endpoint implementation returning a list of all source filenames to test for highlighting using cypress. """ _init_content_code_data_filenames() return Response( sorted(_content_code_data_filenames.keys()), status=200, content_type="application/json", ) @api_view(["GET"]) def get_content_code_data_by_filename(request, filename): """ Endpoint implementation returning metadata of a code content example based on the source filename. """ data = None status = 404 _init_content_code_data_filenames() if filename in _content_code_data_filenames: data = _content_code_data_filenames[filename] status = 200 return Response(data, status=status, content_type="application/json")