Changeset View
Changeset View
Standalone View
Standalone View
swh/web/api/apidoc.py
Show All 18 Lines | |||||
from swh.web.api.apiresponse import make_api_response | from swh.web.api.apiresponse import make_api_response | ||||
from swh.web.api.apiurls import APIUrls | from swh.web.api.apiurls import APIUrls | ||||
from swh.web.common.utils import parse_rst | from swh.web.common.utils import parse_rst | ||||
class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): | class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): | ||||
""" | """ | ||||
docutils visitor for walking on a parsed rst document containing sphinx | docutils visitor for walking on a parsed docutils document containing sphinx | ||||
httpdomain roles. Its purpose is to extract relevant info regarding swh | httpdomain roles. Its purpose is to extract relevant info regarding swh | ||||
api endpoints (for instance url arguments) from their docstring written | api endpoints (for instance url arguments) from their docstring written | ||||
using sphinx httpdomain. | using sphinx httpdomain; and produce the main description back into a ReST | ||||
string | |||||
""" | """ | ||||
# httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) | # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) | ||||
parameter_roles = ("param", "parameter", "arg", "argument") | parameter_roles = ("param", "parameter", "arg", "argument") | ||||
request_json_object_roles = ("reqjsonobj", "reqjson", "<jsonobj", "<json") | request_json_object_roles = ("reqjsonobj", "reqjson", "<jsonobj", "<json") | ||||
request_json_array_roles = ("reqjsonarr", "<jsonarr") | request_json_array_roles = ("reqjsonarr", "<jsonarr") | ||||
Show All 18 Lines | def __init__(self, document, data): | ||||
self.inputs_set = set() | self.inputs_set = set() | ||||
self.returns_set = set() | self.returns_set = set() | ||||
self.status_codes_set = set() | self.status_codes_set = set() | ||||
self.reqheaders_set = set() | self.reqheaders_set = set() | ||||
self.resheaders_set = set() | self.resheaders_set = set() | ||||
self.field_list_visited = False | self.field_list_visited = False | ||||
self.current_json_obj = None | self.current_json_obj = None | ||||
def process_paragraph(self, par): | def _default_visit(self, node: docutils.nodes.Element) -> str: | ||||
""" | """Simply visits a text node, drops its start and end tags, visits | ||||
Process extracted paragraph text before display. | the children, and concatenates their results.""" | ||||
Cleanup document model markups and transform the | return "".join(map(self.dispatch_visit, node.children)) | ||||
paragraph into a valid raw rst string (as the apidoc | |||||
documentation transform rst to html when rendering). | def visit_emphasis(self, node: docutils.nodes.emphasis) -> str: | ||||
""" | return f"*{self._default_visit(node)}*" | ||||
par = par.replace("\n", " ") | |||||
# keep emphasized, strong and literal text | def visit_strong(self, node: docutils.nodes.emphasis) -> str: | ||||
par = par.replace("<emphasis>", "*") | return f"**{self._default_visit(node)}**" | ||||
par = par.replace("</emphasis>", "*") | |||||
par = par.replace("<strong>", "**") | def visit_reference(self, node: docutils.nodes.reference) -> str: | ||||
par = par.replace("</strong>", "**") | text = self._default_visit(node) | ||||
par = par.replace("<literal>", "``") | refuri = node.attributes.get("refuri") | ||||
par = par.replace("</literal>", "``") | if refuri is not None: | ||||
# keep links to web pages | return f"`{text} <{refuri}>`__" | ||||
if "<reference" in par: | else: | ||||
par = re.sub( | return f"`{text}`_" | ||||
r'<reference name="(.*)" refuri="(.*)".*</reference>', | |||||
r"`\1 <\2>`_", | def visit_target(self, node: docutils.nodes.reference) -> str: | ||||
par, | parts = ["\n"] | ||||
parts.extend( | |||||
f".. _{name}: {node.attributes['refuri']}" | |||||
for name in node.attributes["names"] | |||||
) | ) | ||||
# remove parsed document markups but keep rst links | return "\n".join(parts) | ||||
par = re.sub(r"<[^<]+?>(?!`_)", "", par) | |||||
# api urls cleanup to generate valid links afterwards | def visit_literal(self, node: docutils.nodes.literal) -> str: | ||||
subs_made = 1 | return f"``{self._default_visit(node)}``" | ||||
while subs_made: | |||||
(par, subs_made) = re.subn(r"(:http:.*)(\(\w+\))", r"\1", par) | |||||
subs_made = 1 | |||||
while subs_made: | |||||
(par, subs_made) = re.subn(r"(:http:.*)(\[.*\])", r"\1", par) | |||||
par = re.sub(r"([^:])//", r"\1/", par) | |||||
# transform references to api endpoints doc into valid rst links | |||||
par = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", par) | |||||
# transform references to some elements into bold text | |||||
par = re.sub(":http:header:`(.*)`", r"**\1**", par) | |||||
par = re.sub(":func:`(.*)`", r"**\1**", par) | |||||
return par | |||||
def visit_field_list(self, node): | def visit_field_list(self, node): | ||||
""" | """ | ||||
Visit parsed rst field lists to extract relevant info | Visit parsed rst field lists to extract relevant info | ||||
regarding api endpoint. | regarding api endpoint. | ||||
""" | """ | ||||
self.field_list_visited = True | self.field_list_visited = True | ||||
for child in node.traverse(): | for child in node.traverse(): | ||||
# TODO: instead of traversing recursively, we should inspect the children | |||||
# directly (they can be <field_name> and <field_body> directly, or | |||||
# a <field> node containing both) | |||||
# get the parsed field name | # get the parsed field name | ||||
if isinstance(child, docutils.nodes.field_name): | if isinstance(child, docutils.nodes.field_name): | ||||
field_name = child.astext() | field_name = child.astext() | ||||
# parse field text | # parse field text | ||||
elif isinstance(child, docutils.nodes.paragraph): | elif isinstance(child, docutils.nodes.field_body): | ||||
text = self.process_paragraph(str(child)) | text = self._default_visit(child).strip() | ||||
assert text, str(child) | |||||
field_data = field_name.split(" ") | field_data = field_name.split(" ") | ||||
# Parameters | # Parameters | ||||
if field_data[0] in self.parameter_roles: | if field_data[0] in self.parameter_roles: | ||||
if field_data[2] not in self.args_set: | if field_data[2] not in self.args_set: | ||||
self.data["args"].append( | self.data["args"].append( | ||||
{"name": field_data[2], "type": field_data[1], "doc": text} | {"name": field_data[2], "type": field_data[1], "doc": text} | ||||
) | ) | ||||
self.args_set.add(field_data[2]) | self.args_set.add(field_data[2]) | ||||
▲ Show 20 Lines • Show All 61 Lines • ▼ Show 20 Lines | def visit_field_list(self, node): | ||||
self.data["resheaders"].append(resheader) | self.data["resheaders"].append(resheader) | ||||
self.resheaders_set.add(field_data[1]) | self.resheaders_set.add(field_data[1]) | ||||
if ( | if ( | ||||
resheader["name"] == "Content-Type" | resheader["name"] == "Content-Type" | ||||
and resheader["doc"] == "application/octet-stream" | and resheader["doc"] == "application/octet-stream" | ||||
): | ): | ||||
self.data["return_type"] = "octet stream" | self.data["return_type"] = "octet stream" | ||||
def visit_paragraph(self, node): | # Don't return anything in the description; these nodes only add text | ||||
# to other fields | |||||
return "" | |||||
# visit_field_list collects and handles these with a more global view: | |||||
visit_field = visit_field_name = visit_field_body = _default_visit | |||||
def visit_paragraph(self, node: docutils.nodes.paragraph) -> str: | |||||
""" | """ | ||||
Visit relevant paragraphs to parse | Visit relevant paragraphs to parse | ||||
""" | """ | ||||
# only parsed top level paragraphs | # only parsed top level paragraphs | ||||
if isinstance(node.parent, docutils.nodes.block_quote): | text = self._default_visit(node) | ||||
text = self.process_paragraph(str(node)) | |||||
# endpoint description | |||||
if not text.startswith("**") and text not in self.data["description"]: | |||||
self.data["description"] += "\n\n" if self.data["description"] else "" | |||||
self.data["description"] += text | |||||
def visit_literal_block(self, node): | return "\n\n" + text | ||||
def visit_literal_block(self, node: docutils.nodes.literal_block) -> str: | |||||
""" | """ | ||||
Visit literal blocks | Visit literal blocks | ||||
""" | """ | ||||
text = node.astext() | text = node.astext() | ||||
# literal block in endpoint description | |||||
if not self.field_list_visited: | |||||
self.data["description"] += ":\n\n%s\n" % textwrap.indent(text, "\t") | |||||
# extract example urls | |||||
if ":swh_web_api:" in text: | |||||
examples_str = re.sub(".*`(.+)`.*", r"/api/1/\1", text) | |||||
self.data["examples"] += examples_str.split("\n") | |||||
def visit_bullet_list(self, node): | return f"\n\n::\n\n{textwrap.indent(text, ' ')}\n" | ||||
# bullet list in endpoint description | |||||
if not self.field_list_visited: | def visit_bullet_list(self, node: docutils.nodes.bullet_list) -> str: | ||||
self.data["description"] += "\n\n" | parts = ["\n\n"] | ||||
for child in node.traverse(): | |||||
# process list item | |||||
if isinstance(child, docutils.nodes.paragraph): | |||||
line_text = self.process_paragraph(str(child)) | |||||
self.data["description"] += "\t* %s\n" % line_text | |||||
elif self.current_json_obj: | |||||
self.current_json_obj["doc"] += "\n\n" | |||||
for child in node.traverse(): | for child in node.traverse(): | ||||
# process list item | # process list item | ||||
if isinstance(child, docutils.nodes.paragraph): | if isinstance(child, docutils.nodes.paragraph): | ||||
line_text = self.process_paragraph(str(child)) | line_text = self.dispatch_visit(child) | ||||
self.current_json_obj["doc"] += "\t\t* %s\n" % line_text | parts.append("\t* %s\n" % textwrap.indent(line_text, "\t ").strip()) | ||||
self.current_json_obj = None | return "".join(parts) | ||||
# visit_bullet_list collects and handles this with a more global view: | |||||
visit_list_item = _default_visit | |||||
def visit_warning(self, node: docutils.nodes.warning) -> str: | |||||
text = self._default_visit(node) | |||||
return "\n\n.. warning::\n%s\n" % textwrap.indent(text, "\t") | |||||
def visit_Text(self, node: docutils.nodes.Text) -> str: | |||||
"""Leaf node""" | |||||
return str(node).replace("\n", " ") # Prettier in generated HTML | |||||
def visit_warning(self, node): | def visit_problematic(self, node: docutils.nodes.problematic) -> str: | ||||
text = self.process_paragraph(str(node)) | # api urls cleanup to generate valid links afterwards | ||||
rst_warning = "\n\n.. warning::\n%s\n" % textwrap.indent(text, "\t") | text = self._default_visit(node) | ||||
if rst_warning not in self.data["description"]: | subs_made = 1 | ||||
self.data["description"] += rst_warning | while subs_made: | ||||
(text, subs_made) = re.subn(r"(:http:.*)(\(\w+\))", r"\1", text) | |||||
subs_made = 1 | |||||
while subs_made: | |||||
(text, subs_made) = re.subn(r"(:http:.*)(\[.*\])", r"\1", text) | |||||
text = re.sub(r"([^:])//", r"\1/", text) | |||||
# transform references to api endpoints doc into valid rst links | |||||
text = re.sub(":http:get:`([^,`]*)`", r"`\1 <\1doc/>`_", text) | |||||
# transform references to some elements into bold text | |||||
text = re.sub(":http:header:`(.*)`", r"**\1**", text) | |||||
text = re.sub(":func:`(.*)`", r"**\1**", text) | |||||
def unknown_visit(self, node): | # extract example urls | ||||
pass | if ":swh_web_api:" in text: | ||||
# Extract examples to their own section | |||||
examples_str = re.sub(":swh_web_api:`(.+)`.*", r"/api/1/\1", text) | |||||
self.data["examples"] += examples_str.split("\n") | |||||
return text | |||||
def visit_block_quote(self, node: docutils.nodes.block_quote) -> str: | |||||
return self._default_visit(node) | |||||
return ( | |||||
f".. code-block::\n" | |||||
f"{textwrap.indent(self._default_visit(node), ' ')}\n" | |||||
) | |||||
def visit_title_reference(self, node: docutils.nodes.title_reference) -> str: | |||||
text = self._default_visit(node) | |||||
raise Exception( | |||||
f"Unexpected title reference. " | |||||
f"Possible cause: you used `{text}` instead of ``{text}``" | |||||
) | |||||
def visit_document(self, node: docutils.nodes.document) -> None: | |||||
text = self._default_visit(node) | |||||
# Strip examples; they are displayed separately | |||||
text = re.split("\n\\*\\*Examples?:\\*\\*\n", text)[0] | |||||
self.data["description"] = text.strip() | |||||
def unknown_visit(self, node) -> str: | |||||
raise NotImplementedError( | |||||
f"Unknown node type: {node.__class__.__name__}. Value: {node}" | |||||
) | |||||
def unknown_departure(self, node): | def unknown_departure(self, node): | ||||
pass | pass | ||||
def _parse_httpdomain_doc(doc, data): | def _parse_httpdomain_doc(doc, data): | ||||
doc_lines = doc.split("\n") | doc_lines = doc.split("\n") | ||||
doc_lines_filtered = [] | doc_lines_filtered = [] | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | ): | ||||
tags_set = set(tags) | tags_set = set(tags) | ||||
# @api_doc() Decorator call | # @api_doc() Decorator call | ||||
def decorator(f): | def decorator(f): | ||||
# if the route is not hidden, add it to the index | # if the route is not hidden, add it to the index | ||||
if "hidden" not in tags_set: | if "hidden" not in tags_set: | ||||
doc_data = get_doc_data(f, route, noargs) | doc_data = get_doc_data(f, route, noargs) | ||||
doc_desc = doc_data["description"] | doc_desc = doc_data["description"] | ||||
first_dot_pos = doc_desc.find(".") | |||||
APIUrls.add_doc_route( | APIUrls.add_doc_route( | ||||
route, | route, | ||||
doc_desc[: first_dot_pos + 1], | re.split(r"\.\s", doc_desc)[0], | ||||
noargs=noargs, | noargs=noargs, | ||||
api_version=api_version, | api_version=api_version, | ||||
tags=tags_set, | tags=tags_set, | ||||
) | ) | ||||
# create a dedicated view to display endpoint HTML doc | # create a dedicated view to display endpoint HTML doc | ||||
@api_view(["GET", "HEAD"]) | @api_view(["GET", "HEAD"]) | ||||
@wraps(f) | @wraps(f) | ||||
▲ Show 20 Lines • Show All 63 Lines • ▼ Show 20 Lines | elif "SWH_DOC_BUILD" not in os.environ: | ||||
returns_list = "" | returns_list = "" | ||||
for inp in data["inputs"]: | for inp in data["inputs"]: | ||||
# special case for array of non object type, for instance | # special case for array of non object type, for instance | ||||
# :<jsonarr string -: an array of string | # :<jsonarr string -: an array of string | ||||
if inp["name"] != "-": | if inp["name"] != "-": | ||||
inputs_list += "\t* **%s (%s)**: %s\n" % ( | inputs_list += "\t* **%s (%s)**: %s\n" % ( | ||||
inp["name"], | inp["name"], | ||||
inp["type"], | inp["type"], | ||||
inp["doc"], | textwrap.indent(inp["doc"], "\t "), | ||||
) | ) | ||||
for ret in data["returns"]: | for ret in data["returns"]: | ||||
# special case for array of non object type, for instance | # special case for array of non object type, for instance | ||||
# :>jsonarr string -: an array of string | # :>jsonarr string -: an array of string | ||||
if ret["name"] != "-": | if ret["name"] != "-": | ||||
returns_list += "\t* **%s (%s)**: %s\n" % ( | returns_list += "\t* **%s (%s)**: %s\n" % ( | ||||
ret["name"], | ret["name"], | ||||
ret["type"], | ret["type"], | ||||
ret["doc"], | textwrap.indent(ret["doc"], "\t "), | ||||
) | ) | ||||
data["inputs_list"] = inputs_list | data["inputs_list"] = inputs_list | ||||
data["returns_list"] = returns_list | data["returns_list"] = returns_list | ||||
return data | return data | ||||
DOC_COMMON_HEADERS = """ | DOC_COMMON_HEADERS = """ | ||||
Show All 21 Lines |