Page MenuHomeSoftware Heritage
Files F9311628

No OneTemporary
Actions

View Options

diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py
index 58f93157..d3294407 100644
--- a/swh/web/api/apiurls.py
+++ b/swh/web/api/apiurls.py
@@ -1,125 +1,125 @@
-# Copyright (C) 2017-2019 The Software Heritage developers
+# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import functools
from typing import Dict, List, Optional
from django.http.response import HttpResponseBase
from rest_framework.decorators import api_view
from swh.web.api import throttling
from swh.web.api.apiresponse import make_api_response
from swh.web.common.urlsindex import UrlsIndex
class APIUrls(UrlsIndex):
"""
Class to manage API documentation URLs.
- Indexes all routes documented using apidoc's decorators.
- Tracks endpoint/request processing method relationships for use in
generating related urls in API documentation
"""
- _apidoc_routes = {} # type: Dict[str, Dict[str, str]]
+ _apidoc_routes: Dict[str, Dict[str, str]] = {}
scope = "api"
@classmethod
def get_app_endpoints(cls) -> Dict[str, Dict[str, str]]:
return cls._apidoc_routes
@classmethod
def add_doc_route(
cls,
route: str,
docstring: str,
noargs: bool = False,
api_version: str = "1",
**kwargs,
) -> None:
"""
Add a route to the self-documenting API reference
"""
route_name = route[1:-1].replace("/", "-")
if not noargs:
route_name = "%s-doc" % route_name
route_view_name = "api-%s-%s" % (api_version, route_name)
if route not in cls._apidoc_routes:
d = {
"docstring": docstring,
"route": "/api/%s%s" % (api_version, route),
"route_view_name": route_view_name,
}
for k, v in kwargs.items():
d[k] = v
cls._apidoc_routes[route] = d
def api_route(
url_pattern: str,
view_name: Optional[str] = None,
methods: List[str] = ["GET", "HEAD", "OPTIONS"],
throttle_scope: str = "swh_api",
api_version: str = "1",
checksum_args: Optional[List[str]] = None,
never_cache: bool = False,
):
"""
Decorator to ease the registration of an API endpoint
using the Django REST Framework.
Args:
url_pattern: the url pattern used by DRF to identify the API route
view_name: the name of the API view associated to the route used to
reverse the url
methods: array of HTTP methods supported by the API route
throttle_scope: Named scope for rate limiting
api_version: web API version
checksum_args: list of view argument names holding checksum values
never_cache: define if api response must be cached
"""
url_pattern = "^" + api_version + url_pattern + "$"
def decorator(f):
# create a DRF view from the wrapped function
@api_view(methods)
@throttling.throttle_scope(throttle_scope)
@functools.wraps(f)
def api_view_f(request, **kwargs):
# never_cache will be handled in apiresponse module
request.never_cache = never_cache
response = f(request, **kwargs)
doc_data = None
# check if response has been forwarded by api_doc decorator
if isinstance(response, dict) and "doc_data" in response:
doc_data = response["doc_data"]
response = response["data"]
# check if HTTP response needs to be created
if not isinstance(response, HttpResponseBase):
api_response = make_api_response(
request, data=response, doc_data=doc_data
)
else:
api_response = response
return api_response
# small hacks for correctly generating API endpoints index doc
api_view_f.__name__ = f.__name__
api_view_f.http_method_names = methods
# register the route and its view in the endpoints index
APIUrls.add_url_pattern(url_pattern, api_view_f, view_name)
if checksum_args:
APIUrls.add_redirect_for_checksum_args(
view_name, [url_pattern], checksum_args
)
return f
return decorator
diff --git a/swh/web/common/highlightjs.py b/swh/web/common/highlightjs.py
index 362bfd92..ee52bcbc 100644
--- a/swh/web/common/highlightjs.py
+++ b/swh/web/common/highlightjs.py
@@ -1,184 +1,184 @@
# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import functools
import json
from typing import Dict
from pygments.lexers import get_all_lexers, get_lexer_for_filename
from django.contrib.staticfiles.finders import find
from swh.web.common.exc import sentry_capture_exception
@functools.lru_cache()
def _hljs_languages_data():
with open(str(find("json/highlightjs-languages.json")), "r") as hljs_languages_file:
return json.load(hljs_languages_file)
# set of languages ids that can be highlighted by highlight.js library
@functools.lru_cache()
def _hljs_languages():
return set(_hljs_languages_data()["languages"])
# languages aliases defined in highlight.js
@functools.lru_cache()
def _hljs_languages_aliases():
language_aliases = _hljs_languages_data()["languages_aliases"]
language_aliases.pop("robots.txt", None)
return {
**language_aliases,
"ml": "ocaml",
"bsl": "1c",
"ep": "mojolicious",
"lc": "livecode",
"p": "parser3",
"pde": "processing",
"rsc": "routeros",
"s": "armasm",
"sl": "rsl",
"4dm": "4d",
"kaos": "chaos",
"dfy": "dafny",
"ejs": "eta",
"nev": "never",
"m": "octave",
"shader": "hlsl",
"fx": "hlsl",
"prg": "xsharp",
"xs": "xsharp",
}
# dictionary mapping pygment lexers to hljs languages
-_pygments_lexer_to_hljs_language = {} # type: Dict[str, str]
+_pygments_lexer_to_hljs_language: Dict[str, str] = {}
# dictionary mapping mime types to hljs languages
_mime_type_to_hljs_language = {
"text/x-c": "c",
"text/x-c++": "cpp",
"text/x-msdos-batch": "dos",
"text/x-lisp": "lisp",
"text/x-shellscript": "bash",
}
# dictionary mapping filenames to hljs languages
_filename_to_hljs_language = {
"cmakelists.txt": "cmake",
".htaccess": "apache",
"httpd.conf": "apache",
"access.log": "accesslog",
"nginx.log": "accesslog",
"resolv.conf": "dns",
"dockerfile": "docker",
"nginx.conf": "nginx",
"pf.conf": "pf",
"robots.txt": "robots-txt",
}
# function to fill the above dictionaries
def _init_pygments_to_hljs_map():
if len(_pygments_lexer_to_hljs_language) == 0:
hljs_languages = _hljs_languages()
hljs_languages_aliases = _hljs_languages_aliases()
for lexer in get_all_lexers():
lexer_name = lexer[0]
lang_aliases = lexer[1]
lang_mime_types = lexer[3]
lang = None
for lang_alias in lang_aliases:
if lang_alias in hljs_languages:
lang = lang_alias
_pygments_lexer_to_hljs_language[lexer_name] = lang_alias
break
if lang_alias in hljs_languages_aliases:
lang = hljs_languages_aliases[lang_alias]
_pygments_lexer_to_hljs_language[lexer_name] = lang_alias
break
if lang:
for lang_mime_type in lang_mime_types:
if lang_mime_type not in _mime_type_to_hljs_language:
_mime_type_to_hljs_language[lang_mime_type] = lang
def get_hljs_language_from_filename(filename):
"""Function that tries to associate a language supported by highlight.js
from a filename.
Args:
filename: input filename
Returns:
highlight.js language id or None if no correspondence has been found
"""
_init_pygments_to_hljs_map()
if filename:
filename_lower = filename.lower()
if filename_lower in _filename_to_hljs_language:
return _filename_to_hljs_language[filename_lower]
if filename_lower in _hljs_languages():
return filename_lower
exts = filename_lower.split(".")
# check if file extension matches an hljs language
# also handle .ext.in cases
for ext in reversed(exts[-2:]):
if ext in _hljs_languages():
return ext
if ext in _hljs_languages_aliases():
return _hljs_languages_aliases()[ext]
# otherwise use Pygments language database
lexer = None
# try to find a Pygment lexer
try:
lexer = get_lexer_for_filename(filename)
except Exception as exc:
sentry_capture_exception(exc)
# if there is a correspondence between the lexer and an hljs
# language, return it
if lexer and lexer.name in _pygments_lexer_to_hljs_language:
return _pygments_lexer_to_hljs_language[lexer.name]
# otherwise, try to find a match between the file extensions
# associated to the lexer and the hljs language aliases
if lexer:
exts = [ext.replace("*.", "") for ext in lexer.filenames]
for ext in exts:
if ext in _hljs_languages_aliases():
return _hljs_languages_aliases()[ext]
return None
def get_hljs_language_from_mime_type(mime_type):
"""Function that tries to associate a language supported by highlight.js
from a mime type.
Args:
mime_type: input mime type
Returns:
highlight.js language id or None if no correspondence has been found
"""
_init_pygments_to_hljs_map()
if mime_type and mime_type in _mime_type_to_hljs_language:
return _mime_type_to_hljs_language[mime_type]
return None
@functools.lru_cache()
def get_supported_languages():
"""
Return the list of programming languages that can be highlighted using the
highlight.js library.
Returns:
List[str]: the list of supported languages
"""
return sorted(list(_hljs_languages()))
diff --git a/swh/web/tests/views.py b/swh/web/tests/views.py
index 3f90d7d5..f175a841 100644
--- a/swh/web/tests/views.py
+++ b/swh/web/tests/views.py
@@ -1,171 +1,171 @@
-# Copyright (C) 2018-2020 The Software Heritage developers
+# Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
# Implement some special endpoints used to provide input tests data
# when executing end to end tests with cypress
import os
from typing import Dict
from rest_framework.decorators import api_view
from rest_framework.response import Response
from swh.model import from_disk
from swh.model.from_disk import DiskBackedContent
from swh.model.hashutil import hash_to_hex
from swh.model.model import Content
from swh.web.common.highlightjs import get_hljs_language_from_filename
from swh.web.tests.data import get_tests_data
-_content_code_data_exts = {} # type: Dict[str, Dict[str, str]]
-_content_code_data_filenames = {} # type: Dict[str, Dict[str, str]]
-_content_other_data_exts = {} # type: Dict[str, Dict[str, str]]
+_content_code_data_exts: Dict[str, Dict[str, str]] = {}
+_content_code_data_filenames: Dict[str, Dict[str, str]] = {}
+_content_other_data_exts: Dict[str, Dict[str, str]] = {}
def _init_content_tests_data(data_path, data_dict, ext_key):
"""
Helper function to read the content of a directory, store it
into a test archive and add some files metadata (sha1 and/or
expected programming language) in a dict.
Args:
data_path (str): path to a directory relative to the tests
folder of swh-web
data_dict (dict): the dict that will store files metadata
ext_key (bool): whether to use file extensions or filenames
as dict keys
"""
test_contents_dir = os.path.join(os.path.dirname(__file__), data_path).encode(
"utf-8"
)
directory = from_disk.Directory.from_disk(path=test_contents_dir)
contents = []
for name, obj_ in directory.items():
obj = obj_.to_model()
if obj.object_type in [Content.object_type, DiskBackedContent.object_type]:
c = obj.with_data().to_dict()
c["status"] = "visible"
sha1 = hash_to_hex(c["sha1"])
if ext_key:
key = name.decode("utf-8").split(".")[-1]
filename = "test." + key
else:
filename = name.decode("utf-8").split("/")[-1]
key = filename
language = get_hljs_language_from_filename(filename)
data_dict[key] = {"sha1": sha1, "language": language}
contents.append(Content.from_dict(c))
storage = get_tests_data()["storage"]
storage.content_add(contents)
def _init_content_code_data_exts():
"""
Fill a global dictionary which maps source file extension to
a code content example.
"""
global _content_code_data_exts
if not _content_code_data_exts:
_init_content_tests_data(
"resources/contents/code/extensions", _content_code_data_exts, True
)
def _init_content_other_data_exts():
"""
Fill a global dictionary which maps a file extension to
a content example.
"""
global _content_other_data_exts
if not _content_other_data_exts:
_init_content_tests_data(
"resources/contents/other/extensions", _content_other_data_exts, True
)
def _init_content_code_data_filenames():
"""
Fill a global dictionary which maps a filename to
a content example.
"""
global _content_code_data_filenames
if not _content_code_data_filenames:
_init_content_tests_data(
"resources/contents/code/filenames", _content_code_data_filenames, False
)
@api_view(["GET"])
def get_content_code_data_all_exts(request):
"""
Endpoint implementation returning a list of all source file
extensions to test for highlighting using cypress.
"""
_init_content_code_data_exts()
return Response(
sorted(_content_code_data_exts.keys()),
status=200,
content_type="application/json",
)
@api_view(["GET"])
def get_content_code_data_by_ext(request, ext):
"""
Endpoint implementation returning metadata of a code content example
based on the source file extension.
"""
data = None
status = 404
_init_content_code_data_exts()
if ext in _content_code_data_exts:
data = _content_code_data_exts[ext]
status = 200
return Response(data, status=status, content_type="application/json")
@api_view(["GET"])
def get_content_other_data_by_ext(request, ext):
"""
Endpoint implementation returning metadata of a content example
based on the file extension.
"""
_init_content_other_data_exts()
data = None
status = 404
if ext in _content_other_data_exts:
data = _content_other_data_exts[ext]
status = 200
return Response(data, status=status, content_type="application/json")
@api_view(["GET"])
def get_content_code_data_all_filenames(request):
"""
Endpoint implementation returning a list of all source filenames
to test for highlighting using cypress.
"""
_init_content_code_data_filenames()
return Response(
sorted(_content_code_data_filenames.keys()),
status=200,
content_type="application/json",
)
@api_view(["GET"])
def get_content_code_data_by_filename(request, filename):
"""
Endpoint implementation returning metadata of a code content example
based on the source filename.
"""
data = None
status = 404
_init_content_code_data_filenames()
if filename in _content_code_data_filenames:
data = _content_code_data_filenames[filename]
status = 200
return Response(data, status=status, content_type="application/json")

File Metadata

Mime Type
text/x-diff
Expires
Thu, Jul 3, 10:25 AM (2 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3247147

Event Timeline

About · Developer information · Legal