diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py
index be5c1b6d..96dd96c0 100644
--- a/swh/web/api/apidoc.py
+++ b/swh/web/api/apidoc.py
@@ -1,475 +1,487 @@
-# Copyright (C) 2015-2020 The Software Heritage developers
+# Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
from collections import defaultdict
import functools
from functools import wraps
import os
import re
import textwrap
from typing import List
import docutils.nodes
import docutils.parsers.rst
import docutils.utils
+from django.shortcuts import redirect
from rest_framework.decorators import api_view
from swh.web.api.apiresponse import make_api_response
from swh.web.api.apiurls import APIUrls, CategoryId
-from swh.web.utils import parse_rst
+from swh.web.utils import parse_rst, reverse
class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor):
"""
docutils visitor for walking on a parsed docutils document containing sphinx
httpdomain roles. Its purpose is to extract relevant info regarding swh
api endpoints (for instance url arguments) from their docstring written
using sphinx httpdomain; and produce the main description back into a ReST
string
"""
# httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6)
parameter_roles = ("param", "parameter", "arg", "argument")
request_json_object_roles = ("reqjsonobj", "reqjson", "<jsonobj", "<json")
request_json_array_roles = ("reqjsonarr", "<jsonarr")
response_json_object_roles = ("resjsonobj", "resjson", ">jsonobj", ">json")
response_json_array_roles = ("resjsonarr", ">jsonarr")
query_parameter_roles = ("queryparameter", "queryparam", "qparam", "query")
request_header_roles = ("<header", "reqheader", "requestheader")
response_header_roles = (">header", "resheader", "responseheader")
status_code_roles = ("statuscode", "status", "code")
def __init__(self, document, data):
super().__init__(document)
self.data = data
self.args_set = set()
self.params_set = set()
self.inputs_set = set()
self.returns_set = set()
self.status_codes_set = set()
self.reqheaders_set = set()
self.resheaders_set = set()
self.current_json_obj = None
self.current_field_name = None
def _default_visit(self, node: docutils.nodes.Element) -> str:
"""Simply visits a text node, drops its start and end tags, visits
the children, and concatenates their results."""
return "".join(map(self.dispatch_visit, node.children))
def visit_emphasis(self, node: docutils.nodes.emphasis) -> str:
return f"*{self._default_visit(node)}*"
def visit_strong(self, node: docutils.nodes.emphasis) -> str:
return f"**{self._default_visit(node)}**"
def visit_reference(self, node: docutils.nodes.reference) -> str:
text = self._default_visit(node)
refuri = node.attributes.get("refuri")
if refuri is not None:
return f"`{text} <{refuri}>`__"
else:
return f"`{text}`_"
def visit_target(self, node: docutils.nodes.reference) -> str:
parts = ["\n"]
parts.extend(
f".. _{name}: {node.attributes['refuri']}"
for name in node.attributes["names"]
)
return "\n".join(parts)
def visit_literal(self, node: docutils.nodes.literal) -> str:
return f"``{self._default_visit(node)}``"
def visit_field_name(self, node: docutils.nodes.field_name) -> str:
self.current_field_name = node.astext()
return ""
def visit_field_body(self, node: docutils.nodes.field_body) -> str:
text = self._default_visit(node).strip()
assert text, str(node)
field_data = self.current_field_name.split(" ")
# Parameters
if field_data[0] in self.parameter_roles:
if field_data[2] not in self.args_set:
self.data["args"].append(
{"name": field_data[2], "type": field_data[1], "doc": text}
)
self.args_set.add(field_data[2])
# Query Parameters
if field_data[0] in self.query_parameter_roles:
if field_data[2] not in self.params_set:
self.data["params"].append(
{"name": field_data[2], "type": field_data[1], "doc": text}
)
self.params_set.add(field_data[2])
# Request data type
if (
field_data[0] in self.request_json_array_roles
or field_data[0] in self.request_json_object_roles
):
# array
if field_data[0] in self.request_json_array_roles:
self.data["input_type"] = "array"
# object
else:
self.data["input_type"] = "object"
# input object field
if field_data[2] not in self.inputs_set:
self.data["inputs"].append(
{"name": field_data[2], "type": field_data[1], "doc": text}
)
self.inputs_set.add(field_data[2])
self.current_json_obj = self.data["inputs"][-1]
# Response type
if (
field_data[0] in self.response_json_array_roles
or field_data[0] in self.response_json_object_roles
):
# array
if field_data[0] in self.response_json_array_roles:
self.data["return_type"] = "array"
# object
else:
self.data["return_type"] = "object"
# returned object field
if field_data[2] not in self.returns_set:
self.data["returns"].append(
{"name": field_data[2], "type": field_data[1], "doc": text}
)
self.returns_set.add(field_data[2])
self.current_json_obj = self.data["returns"][-1]
# Status Codes
if field_data[0] in self.status_code_roles:
if field_data[1] not in self.status_codes_set:
self.data["status_codes"].append({"code": field_data[1], "doc": text})
self.status_codes_set.add(field_data[1])
# Request Headers
if field_data[0] in self.request_header_roles:
if field_data[1] not in self.reqheaders_set:
self.data["reqheaders"].append({"name": field_data[1], "doc": text})
self.reqheaders_set.add(field_data[1])
# Response Headers
if field_data[0] in self.response_header_roles:
if field_data[1] not in self.resheaders_set:
resheader = {"name": field_data[1], "doc": text}
self.data["resheaders"].append(resheader)
self.resheaders_set.add(field_data[1])
if (
resheader["name"] == "Content-Type"
and resheader["doc"] == "application/octet-stream"
):
self.data["return_type"] = "octet stream"
# Don't return anything in the description; these nodes only add text
# to other fields
return ""
# We ignore these nodes and handle their subtrees directly in
# visit_field_name and visit_field_body
visit_field = visit_field_list = _default_visit
def visit_paragraph(self, node: docutils.nodes.paragraph) -> str:
"""
Visit relevant paragraphs to parse
"""
# only parsed top level paragraphs
text = self._default_visit(node)
return "\n\n" + text
def visit_literal_block(self, node: docutils.nodes.literal_block) -> str:
"""
Visit literal blocks
"""
text = node.astext()
return f"\n\n::\n\n{textwrap.indent(text, ' ')}\n"
def visit_bullet_list(self, node: docutils.nodes.bullet_list) -> str:
parts = ["\n\n"]
for child in node.traverse():
# process list item
if isinstance(child, docutils.nodes.paragraph):
line_text = self.dispatch_visit(child)
parts.append("\t* %s\n" % textwrap.indent(line_text, "\t ").strip())
return "".join(parts)
# visit_bullet_list collects and handles this with a more global view:
visit_list_item = _default_visit
def visit_warning(self, node: docutils.nodes.warning) -> str:
text = self._default_visit(node)
return "\n\n.. warning::\n%s\n" % textwrap.indent(text, "\t")
def visit_Text(self, node: docutils.nodes.Text) -> str:
"""Leaf node"""
return str(node).replace("\n", " ") # Prettier in generated HTML
def visit_problematic(self, node: docutils.nodes.problematic) -> str:
# api urls cleanup to generate valid links afterwards
text = self._default_visit(node)
subs_made = 1
while subs_made:
(text, subs_made) = re.subn(r"(:http:.*)(\(\w+\))", r"\1", text)
subs_made = 1
while subs_made:
(text, subs_made) = re.subn(r"(:http:.*)(\[.*\])", r"\1", text)
text = re.sub(r"([^:])//", r"\1/", text)
# transform references to api endpoints doc into valid rst links
text = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", text)
# transform references to some elements into bold text
text = re.sub(":http:header:`(.*)`", r"**\1**", text)
text = re.sub(":func:`(.*)`", r"**\1**", text)
text = re.sub(":mod:`(.*)`", r"**\1**", text)
# extract example urls
if ":swh_web_api:" in text:
# Extract examples to their own section
examples_str = re.sub(":swh_web_api:`(.+)`.*", r"/api/1/\1", text)
self.data["examples"] += examples_str.split("\n")
return text
def visit_block_quote(self, node: docutils.nodes.block_quote) -> str:
return self._default_visit(node)
return (
f".. code-block::\n"
f"{textwrap.indent(self._default_visit(node), ' ')}\n"
)
def visit_title_reference(self, node: docutils.nodes.title_reference) -> str:
text = self._default_visit(node)
raise Exception(
f"Unexpected title reference. "
f"Possible cause: you used `{text}` instead of ``{text}``"
)
def visit_document(self, node: docutils.nodes.document) -> None:
text = self._default_visit(node)
# Strip examples; they are displayed separately
text = re.split("\n\\*\\*Examples?:\\*\\*\n", text)[0]
self.data["description"] = text.strip()
def visit_system_message(self, node):
return ""
def unknown_visit(self, node) -> str:
raise NotImplementedError(
f"Unknown node type: {node.__class__.__name__}. Value: {node}"
)
def unknown_departure(self, node):
pass
def _parse_httpdomain_doc(doc, data):
doc_lines = doc.split("\n")
doc_lines_filtered = []
urls = defaultdict(list)
default_http_methods = ["HEAD", "OPTIONS"]
# httpdomain is a sphinx extension that is unknown to docutils but
# fortunately we can still parse its directives' content,
# so remove lines with httpdomain directives before executing the
# rst parser from docutils
for doc_line in doc_lines:
if ".. http" not in doc_line:
doc_lines_filtered.append(doc_line)
else:
url = doc_line[doc_line.find("/") :]
# emphasize url arguments for html rendering
url = re.sub(r"\((\w+)\)", r" **\(\1\)** ", url)
method = re.search(r"http:(\w+)::", doc_line).group(1)
urls[url].append(method.upper())
for url, methods in urls.items():
data["urls"].append({"rule": url, "methods": methods + default_http_methods})
# parse the rst docstring and do not print system messages about
# unknown httpdomain roles
document = parse_rst("\n".join(doc_lines_filtered), report_level=5)
# remove the system_message nodes from the parsed document
for node in document.traverse(docutils.nodes.system_message):
node.parent.remove(node)
# visit the document nodes to extract relevant endpoint info
visitor = _HTTPDomainDocVisitor(document, data)
document.walkabout(visitor)
class APIDocException(Exception):
"""
Custom exception to signal errors in the use of the APIDoc decorators
"""
def api_doc(
route: str,
*,
category: CategoryId,
noargs: bool = False,
tags: List[str] = [],
api_version: str = "1",
):
"""
Decorator for an API endpoint implementation used to generate a dedicated
view displaying its HTML documentation.
The documentation will be generated from the endpoint docstring based on
sphinxcontrib-httpdomain format.
Args:
route: documentation page's route
noargs: set to True if the route has no arguments, and its
result should be displayed anytime its documentation
is requested. Default to False
tags: Further information on api endpoints. Two values are
possibly expected:
* hidden: remove the entry points from the listing
* upcoming: display the entry point but it is not followable
* deprecated: display the entry point as deprecated in the index
api_version: api version string
"""
tags_set = set(tags)
# @api_doc() Decorator call
def decorator(f):
# if the route is not hidden, add it to the index
if "hidden" not in tags_set:
doc_data = get_doc_data(f, route, noargs)
doc_desc = doc_data["description"]
APIUrls.add_doc_route(
route,
category,
re.split(r"\.\s", doc_desc)[0],
noargs=noargs,
api_version=api_version,
tags=tags_set,
)
# create a dedicated view to display endpoint HTML doc
@api_view(["GET", "HEAD"])
@wraps(f)
def doc_view(request):
doc_data = get_doc_data(f, route, noargs)
return make_api_response(request, None, doc_data)
route_name = "%s-doc" % route[1:-1].replace("/", "-")
- urlpattern = f"^{api_version}{route}doc/$"
+ urlpattern = f"^api/{api_version}{route}doc/$"
view_name = "api-%s-%s" % (api_version, route_name)
APIUrls.add_url_pattern(urlpattern, doc_view, view_name)
+ # for backward compatibility as previous apidoc URLs were missing
+ # the /api prefix
+ old_view_name = view_name.replace("api-", "")
+ old_urlpattern = f"^{api_version}{route}doc/$"
+
+ @api_view(["GET", "HEAD"])
+ def old_doc_view(request):
+ return redirect(reverse(view_name))
+
+ APIUrls.add_url_pattern(old_urlpattern, old_doc_view, old_view_name)
+
@wraps(f)
def documented_view(request, **kwargs):
doc_data = get_doc_data(f, route, noargs)
try:
return {"data": f(request, **kwargs), "doc_data": doc_data}
except Exception as exc:
exc.doc_data = doc_data
raise exc
return documented_view
return decorator
@functools.lru_cache(maxsize=32)
def get_doc_data(f, route, noargs):
"""
Build documentation data for the decorated api endpoint function
"""
data = {
"description": "",
"response_data": None,
"urls": [],
"args": [],
"params": [],
"input_type": "",
"inputs": [],
"resheaders": [],
"reqheaders": [],
"return_type": "",
"returns": [],
"status_codes": [],
"examples": [],
"route": route,
"noargs": noargs,
}
if not f.__doc__:
raise APIDocException(
"apidoc: expected a docstring" " for function %s" % (f.__name__,)
)
# use raw docstring as endpoint documentation if sphinx
# httpdomain is not used
if ".. http" not in f.__doc__:
data["description"] = f.__doc__
# else parse the sphinx httpdomain docstring with docutils
# (except when building the swh-web documentation through autodoc
# sphinx extension, not needed and raise errors with sphinx >= 1.7)
elif "SWH_DOC_BUILD" not in os.environ:
_parse_httpdomain_doc(f.__doc__, data)
# process input/returned object info for nicer html display
inputs_list = ""
returns_list = ""
for inp in data["inputs"]:
# special case for array of non object type, for instance
# :<jsonarr string -: an array of string
if inp["name"] != "-":
inputs_list += "\t* **%s (%s)**: %s\n" % (
inp["name"],
inp["type"],
textwrap.indent(inp["doc"], "\t "),
)
for ret in data["returns"]:
# special case for array of non object type, for instance
# :>jsonarr string -: an array of string
if ret["name"] != "-":
returns_list += "\t* **%s (%s)**: %s\n" % (
ret["name"],
ret["type"],
textwrap.indent(ret["doc"], "\t "),
)
data["inputs_list"] = inputs_list
data["returns_list"] = returns_list
return data
DOC_COMMON_HEADERS = """
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept`
header of request"""
DOC_RESHEADER_LINK = """
:resheader Link: indicates that a subsequent result page is
available and contains the url pointing to it
"""
DEFAULT_SUBSTITUTIONS = {
"common_headers": DOC_COMMON_HEADERS,
"resheader_link": DOC_RESHEADER_LINK,
}
def format_docstring(**substitutions):
def decorator(f):
f.__doc__ = f.__doc__.format(**{**DEFAULT_SUBSTITUTIONS, **substitutions})
return f
return decorator
diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py
index 2c90205f..3fe2ba6c 100644
--- a/swh/web/api/apiresponse.py
+++ b/swh/web/api/apiresponse.py
@@ -1,230 +1,238 @@
-# Copyright (C) 2017-2021 The Software Heritage developers
+# Copyright (C) 2017-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import json
import logging
import traceback
from typing import Any, Dict, Optional
from django.http import HttpResponse
from django.shortcuts import render
+from django.urls import get_resolver
from django.utils.cache import add_never_cache_headers
from django.utils.html import escape
from rest_framework.exceptions import APIException
from rest_framework.request import Request
from rest_framework.response import Response
from rest_framework.utils.encoders import JSONEncoder
from swh.storage.exc import StorageAPIError, StorageDBError
from swh.web.api import utils
from swh.web.config import get_config
from swh.web.utils import gen_path_info, shorten_path
from swh.web.utils.exc import (
BadInputExc,
ForbiddenExc,
LargePayloadExc,
NotFoundExc,
sentry_capture_exception,
)
logger = logging.getLogger("django")
def compute_link_header(rv: Dict[str, Any], options: Dict[str, Any]) -> Dict[str, Any]:
"""Add Link header in returned value results.
Args:
request: a DRF Request object
rv (dict): dictionary with keys:
- headers: potential headers with 'link-next' and 'link-prev'
keys
- results: containing the result to return
options (dict): the initial dict to update with result if any
Returns:
dict: dictionary with optional keys 'link-next' and 'link-prev'
"""
link_headers = []
if "headers" not in rv:
return {}
rv_headers = rv["headers"]
if "link-next" in rv_headers:
link_headers.append('<%s>; rel="next"' % rv_headers["link-next"])
if "link-prev" in rv_headers:
link_headers.append('<%s>; rel="previous"' % rv_headers["link-prev"])
if link_headers:
link_header_str = ",".join(link_headers)
headers = options.get("headers", {})
headers.update({"Link": link_header_str})
return headers
return {}
def filter_by_fields(request: Request, data: Dict[str, Any]) -> Dict[str, Any]:
"""Extract a request parameter 'fields' if it exists to permit the filtering on
the data dict's keys.
If such field is not provided, returns the data as is.
"""
fields = request.query_params.get("fields")
if fields:
data = utils.filter_field_keys(data, set(fields.split(",")))
return data
def transform(rv: Dict[str, Any]) -> Dict[str, Any]:
"""Transform an eventual returned value with multiple layer of
information with only what's necessary.
If the returned value rv contains the 'results' key, this is the
associated value which is returned.
Otherwise, return the initial dict without the potential 'headers'
key.
"""
if "results" in rv:
return rv["results"]
if "headers" in rv:
rv.pop("headers")
return rv
def make_api_response(
request: Request,
data: Dict[str, Any],
doc_data: Optional[Dict[str, Any]] = None,
options: Optional[Dict[str, Any]] = None,
) -> HttpResponse:
"""Generates an API response based on the requested mimetype.
Args:
request: a DRF Request object
data: raw data to return in the API response
doc_data: documentation data for HTML response
options: optional data that can be used to generate the response
Returns:
a DRF Response a object
"""
options = options or {}
if data:
options["headers"] = compute_link_header(data, options)
data = transform(data)
data = filter_by_fields(request, data)
doc_data = doc_data or {}
headers = {}
if "headers" in options:
doc_data["headers_data"] = options["headers"]
headers = options["headers"]
# get request status code
doc_data["status_code"] = options.get("status", 200)
accepted_media_type = getattr(request, "accepted_media_type", "application/json")
# when requesting HTML, typically when browsing the API through its
# documented views, we need to enrich the input data with documentation
# and render the apidoc HTML template
if accepted_media_type == "text/html":
doc_data["response_data"] = data
if data is not None:
doc_data["response_data"] = json.dumps(
data, cls=JSONEncoder, sort_keys=True, indent=4, separators=(",", ": ")
)
doc_data["heading"] = shorten_path(str(request.path))
# generate breadcrumbs data
if "route" in doc_data:
+ all_view_names = set(get_resolver().reverse_dict.keys())
doc_data["endpoint_path"] = gen_path_info(doc_data["route"])
for i in range(len(doc_data["endpoint_path"]) - 1):
- doc_data["endpoint_path"][i]["path"] += "/doc/"
+ view_name = "api-1-" + "-".join(
+ [doc_data["endpoint_path"][i]["name"] for i in range(i + 1)]
+ )
+ if view_name in all_view_names:
+ doc_data["endpoint_path"][i]["path"] += "/doc/"
+ else:
+ doc_data["endpoint_path"][i]["path"] = ""
if not doc_data["noargs"]:
doc_data["endpoint_path"][-1]["path"] += "/doc/"
response = render(
request, "apidoc.html", doc_data, status=doc_data["status_code"]
)
# otherwise simply return the raw data and let DRF picks
# the correct renderer (JSON or YAML)
else:
response = Response(
data,
headers=headers,
content_type=accepted_media_type,
status=doc_data["status_code"],
)
if getattr(request, "never_cache", False):
add_never_cache_headers(response)
return response
def error_response(
request: Request, exception: Exception, doc_data: Dict[str, Any]
) -> HttpResponse:
"""Private function to create a custom error response.
Args:
request: a DRF Request object
error: the exception that caused the error
doc_data: documentation data for HTML response
"""
error_code = 500
if isinstance(exception, BadInputExc):
error_code = 400
elif isinstance(exception, NotFoundExc):
error_code = 404
elif isinstance(exception, ForbiddenExc):
error_code = 403
elif isinstance(exception, LargePayloadExc):
error_code = 413
elif isinstance(exception, StorageDBError):
error_code = 503
elif isinstance(exception, StorageAPIError):
error_code = 503
elif isinstance(exception, APIException):
error_code = exception.status_code
error_opts = {"status": error_code}
error_data = {
"exception": exception.__class__.__name__,
"reason": str(exception),
}
if getattr(request, "accepted_media_type", None) == "text/html":
error_data["reason"] = escape(error_data["reason"])
if get_config()["debug"]:
error_data["traceback"] = traceback.format_exc()
logger.debug(error_data["traceback"])
return make_api_response(request, error_data, doc_data, options=error_opts)
def error_response_handler(
exc: Exception, context: Dict[str, Any]
) -> Optional[HttpResponse]:
"""Custom DRF exception handler used to generate API error responses."""
sentry_capture_exception(exc)
doc_data = getattr(exc, "doc_data", {})
return error_response(context["request"], exc, doc_data)
diff --git a/swh/web/api/templates/apidoc.html b/swh/web/api/templates/apidoc.html
index 24e26408..9e13f54f 100644
--- a/swh/web/api/templates/apidoc.html
+++ b/swh/web/api/templates/apidoc.html
@@ -1,221 +1,221 @@
{% extends "layout.html" %}
{% comment %}
-Copyright (C) 2015-2020 The Software Heritage developers
+Copyright (C) 2015-2022 The Software Heritage developers
See the AUTHORS file at the top-level directory of this distribution
License: GNU Affero General Public License version 3, or any later version
See top-level LICENSE file for more information
{% endcomment %}
{% load swh_templatetags %}
{% block title %}{{ heading }} – Software Heritage API {% endblock %}
{% block navbar-content %}
<nav class="bread-crumbs">
<ul>
<li><a href="/api/"><h4>Web API</h4></a></li>
<li class="bc-no-root"><i class="mdi mdi-menu-right mdi-fw" aria-hidden="true"></i></li>
<li class="bc-no-root"><a href="/api/1/">endpoints</a></li>
{% for endpoint in endpoint_path %}
<li class="bc-no-root"><i class="mdi mdi-menu-right mdi-fw" aria-hidden="true"></i></li>
- {% if endpoint.name != 'stat' and endpoint.name != 'vault' and endpoint.path != 'vault/revision/doc/' %}
+ {% if endpoint.path %}
<li class="bc-no-root"><a href="{{ '/api/1/'|add:endpoint.path }}">{{ endpoint.name }}</a></li>
{% else %}
<li class="bc-no-root"><span>{{ endpoint.name }}</span></li>
{% endif %}
{% endfor %}
</ul>
</nav>
{% endblock %}
{% block content %}
<div class='swh-apidoc'>
{% if description %}
<div>
<h4> Description </h4>
{{ description | docstring_display | safe }}
</div>
{% endif %}
{% if response_data is not None %}
<div>
<h4>Request</h4>
<pre><strong>{{ request.method }}</strong> {{ request.build_absolute_uri }}</pre>
<hr />
<h4>Response</h4>
{% if status_code != 200 %}
<h5>Status Code</h5>
<pre>{{ status_code }}</pre>
{% endif %}
{% if headers_data %}
<h5>Headers</h5>
{% for header_name, header_value in headers_data.items %}
<pre><strong>{{ header_name }}</strong> {{ header_value | urlize_header_links | safe }}</pre>
{% endfor %}
{% endif %}
<h5>Body</h5>
<pre><code class="json">{{ response_data | urlize_links_and_mails | safe }}</code></pre>
</div>
{% endif %}
<hr />
{% if urls and urls|length > 0 %}
<div>
<table class="m-x-auto table">
<thead>
<tr>
<th>URL</th>
<th>Allowed Methods</th>
</tr>
</thead>
<tbody>
{% for url in urls %}
<tr>
<td>{{ url.rule | docstring_display | safe }}</td>
<td>{{ url.methods | dictsort:0 | join:', ' }}</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
<hr />
{% endif %}
{% if args and args|length > 0 %}
<div>
<h4> Arguments </h4>
{% for arg in args %}
<dl class="row">
<dt class="col col-md-2 text-right"> {{ arg.name }} ({{ arg.type }}) </dt>
<dd class="col col-md-9"> {{ arg.doc | docstring_display | safe }} </dd>
</dl>
{% endfor %}
</div>
<hr />
{% endif %}
{% if params and params|length > 0 %}
<div>
<h4> Query parameters </h4>
{% for param in params %}
<dl class="row">
<dt class="col col-md-2 text-right"> {{ param.name }} ({{ param.type }}) </dt>
<dd class="col col-md-9"> {{ param.doc | docstring_display | safe }} </dd>
</dl>
{% endfor %}
</div>
<hr />
{% endif %}
{% if reqheaders and reqheaders|length > 0 %}
<div>
<h4> Request headers </h4>
{% for header in reqheaders %}
<dl class="row">
<dt class="col col-md-2 text-right"> {{ header.name }} </dt>
<dd class="col col-md-9"> {{ header.doc | docstring_display | safe }} </dd>
</dl>
{% endfor %}
</div>
<hr />
{% endif %}
{% if input_type %}
<div>
<h4> Request data </h4>
<dl class="row">
<dt class="col col-md-2 text-right"> {{ input_type }} </dt>
<dd class="col col-md-9">
<p>
{% if input_type == 'array' and inputs_list == '' %}
{{ inputs.0.doc | safe }}
{% elif input_type == 'array' and inputs_list != '' %}
an array of objects containing the following keys:
{% elif input_type == 'octet stream' %}
raw data as an octet stream
{% elif input_type == 'object' %}
an object containing the following keys:
{% endif %}
{% if inputs_list != '' %}
{{ inputs_list | docstring_display | safe }}
{% endif %}
</p>
</dd>
</dl>
</div>
<hr />
{% endif %}
{% if resheaders and resheaders|length > 0 %}
<div>
<h4> Response headers </h4>
{% for header in resheaders %}
<dl class="row">
<dt class="col col-md-2 text-right"> {{ header.name }} </dt>
<dd class="col col-md-9"> {{ header.doc | docstring_display | safe }} </dd>
</dl>
{% endfor %}
</div>
<hr />
{% endif %}
{% if return_type %}
<div>
<h4> Returns </h4>
<dl class="row">
<dt class="col col-md-2 text-right"> {{ return_type }} </dt>
<dd class="col col-md-9">
<p>
{% if return_type == 'array' and returns_list == '' %}
{{ returns.0.doc | safe }}
{% elif return_type == 'array' and returns_list != '' %}
an array of objects containing the following keys:
{% elif return_type == 'octet stream' %}
raw data as an octet stream
{% elif return_type == 'object' %}
an object containing the following keys:
{% endif %}
{% if returns_list != '' %}
{{ returns_list | docstring_display | safe }}
{% endif %}
</p>
</dd>
</dl>
</div>
<hr />
{% endif %}
{% if status_codes and status_codes|length > 0 %}
<div>
<h4> HTTP status codes </h4>
{% for status in status_codes %}
<dl class="row">
<dt class="col col-md-2 text-right"> {{ status.code }} </dt>
<dd class="col col-md-9"> {{ status.doc | docstring_display | safe }} </dd>
</dl>
{% endfor %}
</div>
<hr />
{% endif %}
{% if examples and examples|length > 0 %}
<div>
<h4> Examples </h4>
{% for example in examples %}
<dl class="row">
<dt class="col col-md-2"></dt>
<dd class="col col-md-9">
<a href="{{ example }}">{{ example }}</a>
</dd>
</dl>
{% endfor %}
</div>
{% endif %}
</div>
<script>
swh.webapp.initPage('api');
swh.webapp.highlightCode(false);
// restore Web API links removed by code highlighting
setTimeout(function() {
$('.hljs-string').each(function(idx, element) {
var text = $(element).text();
if (text.match(/^"http.*:\/\/.*/)) {
$(element).html('<a class="hljs-string" href=' + text + '>' + text + '</a>')
}
});
}, 500);
</script>
{% endblock %}
diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py
index ae161774..43f072d9 100644
--- a/swh/web/tests/api/test_apidoc.py
+++ b/swh/web/tests/api/test_apidoc.py
@@ -1,481 +1,498 @@
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import textwrap
import pytest
from rest_framework.response import Response
from swh.storage.exc import StorageAPIError, StorageDBError
from swh.web.api.apidoc import _parse_httpdomain_doc, api_doc
from swh.web.api.apiurls import api_route
+from swh.web.tests.django_asserts import assert_contains, assert_not_contains
from swh.web.tests.helpers import check_api_get_responses, check_html_get_response
from swh.web.utils import prettify_html, reverse
from swh.web.utils.exc import BadInputExc, ForbiddenExc, NotFoundExc
_httpdomain_doc = """
.. http:get:: /api/1/revision/(sha1_git)/
Get information about a revision in the archive.
Revisions are identified by **sha1** checksums, compatible with Git commit
identifiers.
See :func:`swh.model.git_objects.revision_git_object` in our data model
module for details about how they are computed.
:param string sha1_git: hexadecimal representation of the revision
**sha1_git** identifier
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header
of request
:<json int n: sample input integer
:<json string s: sample input string
:<json array a: sample input array
:>json object author: information about the author of the revision
:>json object committer: information about the committer of the revision
:>json string committer_date: RFC3339 representation of the commit date
:>json string date: RFC3339 representation of the revision date
:>json string directory: the unique identifier that revision points to
:>json string directory_url: link to
:http:get:`/api/1/directory/(sha1_git)/[(path)/]` to get information
about the directory associated to the revision
:>json string id: the revision unique identifier
:>json boolean merge: whether or not the revision corresponds to a merge
commit
:>json string message: the message associated to the revision
:>json array parents: the parents of the revision, i.e. the previous
revisions that head directly to it, each entry of that array contains
an unique parent revision identifier but also a link to
:http:get:`/api/1/revision/(sha1_git)/` to get more information
about it
:>json string type: the type of the revision
:statuscode 200: no error
:statuscode 400: an invalid **sha1_git** value has been provided
:statuscode 404: requested revision can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/`
"""
_exception_http_code = {
BadInputExc: 400,
ForbiddenExc: 403,
NotFoundExc: 404,
Exception: 500,
StorageAPIError: 503,
StorageDBError: 503,
}
def test_apidoc_nodoc_failure():
with pytest.raises(Exception):
@api_doc("/my/nodoc/url/", "test")
def apidoc_nodoc_tester(request, arga=0, argb=0):
return Response(arga + argb)
@api_route(r"/some/(?P<myarg>[0-9]+)/(?P<myotherarg>[0-9]+)/", "api-1-some-doc-route")
@api_doc("/some/doc/route/", category="test")
def apidoc_route(request, myarg, myotherarg, akw=0):
"""
Sample doc
"""
return {"result": int(myarg) + int(myotherarg) + akw}
def test_apidoc_route_doc(client):
- url = reverse("api-1-some-doc-route-doc")
- check_html_get_response(client, url, status_code=200, template_used="apidoc.html")
+ api_view_name = "api-1-some-doc-route-doc"
+ doc_url = reverse(api_view_name)
+ assert doc_url == "/api/1/some/doc/route/doc/"
+ resp = check_html_get_response(
+ client, doc_url, status_code=200, template_used="apidoc.html"
+ )
+
+ # check apidoc breadcrumbs links
+ api_view_name_split = api_view_name.split("-")
+ for i in range(2, len(api_view_name_split) - 1):
+ sub_doc_url = "/" + ("/".join(api_view_name_split[:i])) + "/doc/"
+ assert_not_contains(resp, f'<a href="{sub_doc_url}">')
+ assert_contains(resp, f'<a href="{doc_url}">')
+
+ # check previous erroneous URL now redirects to the fixed one
+ url = reverse("1-some-doc-route-doc")
+ resp = check_html_get_response(client, url, status_code=302)
+ assert resp["location"] == doc_url
def test_apidoc_route_fn(api_client):
url = reverse("api-1-some-doc-route", url_args={"myarg": 1, "myotherarg": 1})
check_api_get_responses(api_client, url, status_code=200)
@api_route(r"/test/error/(?P<exc_name>.+)/", "api-1-test-error")
@api_doc("/test/error/", category="test")
def apidoc_test_error_route(request, exc_name):
"""
Sample doc
"""
for e in _exception_http_code.keys():
if e.__name__ == exc_name:
raise e("Error")
def test_apidoc_error(api_client):
for exc, code in _exception_http_code.items():
url = reverse("api-1-test-error", url_args={"exc_name": exc.__name__})
check_api_get_responses(api_client, url, status_code=code)
@api_route(
r"/some/full/(?P<myarg>[0-9]+)/(?P<myotherarg>[0-9]+)/",
"api-1-some-complete-doc-route",
)
@api_doc("/some/complete/doc/route/", category="test")
def apidoc_full_stack(request, myarg, myotherarg, akw=0):
"""
Sample doc
"""
return {"result": int(myarg) + int(myotherarg) + akw}
def test_apidoc_full_stack_doc(client):
url = reverse("api-1-some-complete-doc-route-doc")
check_html_get_response(client, url, status_code=200, template_used="apidoc.html")
def test_apidoc_full_stack_fn(api_client):
url = reverse(
"api-1-some-complete-doc-route", url_args={"myarg": 1, "myotherarg": 1}
)
check_api_get_responses(api_client, url, status_code=200)
@api_route(r"/test/post/only/", "api-1-test-post-only", methods=["POST"])
@api_doc("/test/post/only/", category="test")
def apidoc_test_post_only(request, exc_name):
"""
Sample doc
"""
return {"result": "some data"}
def test_apidoc_post_only(client):
# a dedicated view accepting GET requests should have
# been created to display the HTML documentation
url = reverse("api-1-test-post-only-doc")
check_html_get_response(client, url, status_code=200, template_used="apidoc.html")
def test_api_doc_parse_httpdomain():
doc_data = {
"description": "",
"urls": [],
"args": [],
"params": [],
"resheaders": [],
"reqheaders": [],
"input_type": "",
"inputs": [],
"return_type": "",
"returns": [],
"status_codes": [],
"examples": [],
}
_parse_httpdomain_doc(_httpdomain_doc, doc_data)
expected_urls = [
{
"rule": "/api/1/revision/ **\\(sha1_git\\)** /",
"methods": ["GET", "HEAD", "OPTIONS"],
}
]
assert "urls" in doc_data
assert doc_data["urls"] == expected_urls
expected_description = (
"Get information about a revision in the archive. "
"Revisions are identified by **sha1** checksums, "
"compatible with Git commit identifiers. See "
"**swh.model.git_objects.revision_git_object** in "
"our data model module for details about how they "
"are computed."
)
assert "description" in doc_data
assert doc_data["description"] == expected_description
expected_args = [
{
"name": "sha1_git",
"type": "string",
"doc": (
"hexadecimal representation of the revision " "**sha1_git** identifier"
),
}
]
assert "args" in doc_data
assert doc_data["args"] == expected_args
expected_params = []
assert "params" in doc_data
assert doc_data["params"] == expected_params
expected_reqheaders = [
{
"doc": (
"the requested response content type, either "
"``application/json`` (default) or ``application/yaml``"
),
"name": "Accept",
}
]
assert "reqheaders" in doc_data
assert doc_data["reqheaders"] == expected_reqheaders
expected_resheaders = [
{"doc": "this depends on **Accept** header of request", "name": "Content-Type"}
]
assert "resheaders" in doc_data
assert doc_data["resheaders"] == expected_resheaders
expected_statuscodes = [
{"code": "200", "doc": "no error"},
{"code": "400", "doc": "an invalid **sha1_git** value has been provided"},
{"code": "404", "doc": "requested revision can not be found in the archive"},
]
assert "status_codes" in doc_data
assert doc_data["status_codes"] == expected_statuscodes
expected_input_type = "object"
assert "input_type" in doc_data
assert doc_data["input_type"] == expected_input_type
expected_inputs = [
{"name": "n", "type": "int", "doc": "sample input integer"},
{"name": "s", "type": "string", "doc": "sample input string"},
{"name": "a", "type": "array", "doc": "sample input array"},
]
assert "inputs" in doc_data
assert doc_data["inputs"] == expected_inputs
expected_return_type = "object"
assert "return_type" in doc_data
assert doc_data["return_type"] == expected_return_type
expected_returns = [
{
"name": "author",
"type": "object",
"doc": "information about the author of the revision",
},
{
"name": "committer",
"type": "object",
"doc": "information about the committer of the revision",
},
{
"name": "committer_date",
"type": "string",
"doc": "RFC3339 representation of the commit date",
},
{
"name": "date",
"type": "string",
"doc": "RFC3339 representation of the revision date",
},
{
"name": "directory",
"type": "string",
"doc": "the unique identifier that revision points to",
},
{
"name": "directory_url",
"type": "string",
"doc": (
"link to `/api/1/directory/ </api/1/directory/doc/>`_ "
"to get information about the directory associated to "
"the revision"
),
},
{"name": "id", "type": "string", "doc": "the revision unique identifier"},
{
"name": "merge",
"type": "boolean",
"doc": "whether or not the revision corresponds to a merge commit",
},
{
"name": "message",
"type": "string",
"doc": "the message associated to the revision",
},
{
"name": "parents",
"type": "array",
"doc": (
"the parents of the revision, i.e. the previous revisions "
"that head directly to it, each entry of that array "
"contains an unique parent revision identifier but also a "
"link to `/api/1/revision/ </api/1/revision/doc/>`_ "
"to get more information about it"
),
},
{"name": "type", "type": "string", "doc": "the type of the revision"},
]
assert "returns" in doc_data
assert doc_data["returns"] == expected_returns
expected_examples = ["/api/1/revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/"]
assert "examples" in doc_data
assert doc_data["examples"] == expected_examples
@api_route(r"/post/endpoint/", "api-1-post-endpoint", methods=["POST"])
@api_doc("/post/endpoint/", category="test")
def apidoc_test_post_endpoint(request):
"""
.. http:post:: /api/1/post/endpoint/
Endpoint documentation
:<jsonarr string -: Input array of SWHIDs
:>json object <swhid>: an object whose keys are input SWHIDs
and values objects with the following keys:
* **known (bool)**: whether the object was found
"""
pass
def test_apidoc_input_output_doc(client):
url = reverse("api-1-post-endpoint-doc")
rv = check_html_get_response(
client, url, status_code=200, template_used="apidoc.html"
)
input_html_doc = textwrap.indent(
(
'<dl class="row">\n'
' <dt class="col col-md-2 text-right">\n'
" array\n"
" </dt>\n"
' <dd class="col col-md-9">\n'
" <p>\n"
" Input array of SWHIDs\n"
" </p>\n"
" </dd>\n"
"</dl>\n"
),
" " * 7,
)
output_html_doc = textwrap.indent(
(
'<dl class="row">\n'
' <dt class="col col-md-2 text-right">\n'
" object\n"
" </dt>\n"
' <dd class="col col-md-9">\n'
" <p>\n"
" an object containing the following keys:\n"
" </p>\n"
' <div class="swh-rst">\n'
" <blockquote>\n"
" <ul>\n"
" <li>\n"
" <p>\n"
" <strong>\n"
" <swhid> (object)\n"
" </strong>\n"
" : an object whose keys are input SWHIDs"
" and values objects with the following keys:\n"
" </p>\n"
" <blockquote>\n"
' <ul class="simple">\n'
" <li>\n"
" <p>\n"
" <strong>\n"
" known (bool)\n"
" </strong>\n"
" : whether the object was found\n"
" </p>\n"
" </li>\n"
" </ul>\n"
" </blockquote>\n"
" </li>\n"
" </ul>\n"
" </blockquote>\n"
" </div>\n"
" </dd>\n"
"</dl>\n"
),
" " * 7,
)
html = prettify_html(rv.content)
assert input_html_doc in html
assert output_html_doc in html
@api_route(r"/endpoint/links/in/doc/", "api-1-endpoint-links-in-doc")
@api_doc("/endpoint/links/in/doc/", category="test")
def apidoc_test_endpoint_with_links_in_doc(request):
"""
.. http:get:: /api/1/post/endpoint/
Endpoint documentation with links to
:http:get:`/api/1/content/[(hash_type):](hash)/`,
:http:get:`/api/1/directory/(sha1_git)/[(path)/]`
and `archive <https://archive.softwareheritage.org>`_.
"""
pass
def test_apidoc_with_links(client):
url = reverse("api-1-endpoint-links-in-doc")
rv = check_html_get_response(
client, url, status_code=200, template_used="apidoc.html"
)
html = prettify_html(rv.content)
first_link = textwrap.indent(
(
'<a class="reference external" href="/api/1/content/doc/">\n'
" /api/1/content/\n"
"</a>"
),
" " * 9,
)
second_link = textwrap.indent(
(
'<a class="reference external" href="/api/1/directory/doc/">\n'
" /api/1/directory/\n"
"</a>"
),
" " * 9,
)
third_link = textwrap.indent(
(
'<a class="reference external" '
'href="https://archive.softwareheritage.org">\n'
" archive\n"
"</a>"
),
" " * 9,
)
assert first_link in html
assert second_link in html
assert third_link in html