diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py index cdec2990..2323a0ee 100644 --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -1,438 +1,453 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import functools from functools import wraps import os import re import textwrap from typing import List import docutils.nodes import docutils.parsers.rst import docutils.utils from rest_framework.decorators import api_view import sentry_sdk from swh.web.common.utils import parse_rst from swh.web.api.apiurls import APIUrls from swh.web.api.apiresponse import make_api_response, error_response class _HTTPDomainDocVisitor(docutils.nodes.NodeVisitor): """ docutils visitor for walking on a parsed rst document containing sphinx httpdomain roles. Its purpose is to extract relevant info regarding swh api endpoints (for instance url arguments) from their docstring written using sphinx httpdomain. """ # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) parameter_roles = ('param', 'parameter', 'arg', 'argument') request_json_object_roles = ('reqjsonobj', 'reqjson', 'jsonobj', '>json') response_json_array_roles = ('resjsonarr', '>jsonarr') query_parameter_roles = ('queryparameter', 'queryparam', 'qparam', 'query') request_header_roles = ('header', 'resheader', 'responseheader') status_code_roles = ('statuscode', 'status', 'code') def __init__(self, document, urls, data): super().__init__(document) self.urls = urls self.url_idx = 0 self.data = data self.args_set = set() self.params_set = set() self.inputs_set = set() self.returns_set = set() self.status_codes_set = set() self.reqheaders_set = set() self.resheaders_set = set() self.field_list_visited = False + self.current_json_obj = None def process_paragraph(self, par): """ Process extracted paragraph text before display. Cleanup document model markups and transform the paragraph into a valid raw rst string (as the apidoc documentation transform rst to html when rendering). """ par = par.replace('\n', ' ') # keep emphasized, strong and literal text par = par.replace('', '*') par = par.replace('', '*') par = par.replace('', '**') par = par.replace('', '**') par = par.replace('', '``') par = par.replace('', '``') # remove parsed document markups par = re.sub('<[^<]+?>', '', par) # api urls cleanup to generate valid links afterwards - par = re.sub(r'\(\w+\)', '', par) - par = re.sub(r'\[.*\]', '', par) + subs_made = 1 + while subs_made: + (par, subs_made) = re.subn(r'(:http:.*)(\(\w+\))', r'\1', par) + subs_made = 1 + while subs_made: + (par, subs_made) = re.subn(r'(:http:.*)(\[.*\])', r'\1', par) par = par.replace('//', '/') # transform references to api endpoints into valid rst links par = re.sub(':http:get:`([^,]*)`', r'`<\1>`_', par) # transform references to some elements into bold text par = re.sub(':http:header:`(.*)`', r'**\1**', par) par = re.sub(':func:`(.*)`', r'**\1**', par) return par def visit_field_list(self, node): """ Visit parsed rst field lists to extract relevant info regarding api endpoint. """ self.field_list_visited = True for child in node.traverse(): # get the parsed field name if isinstance(child, docutils.nodes.field_name): field_name = child.astext() # parse field text elif isinstance(child, docutils.nodes.paragraph): text = self.process_paragraph(str(child)) field_data = field_name.split(' ') # Parameters if field_data[0] in self.parameter_roles: if field_data[2] not in self.args_set: self.data['args'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.args_set.add(field_data[2]) # Query Parameters if field_data[0] in self.query_parameter_roles: if field_data[2] not in self.params_set: self.data['params'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.params_set.add(field_data[2]) # Request data type if (field_data[0] in self.request_json_array_roles or field_data[0] in self.request_json_object_roles): # array if field_data[0] in self.request_json_array_roles: self.data['input_type'] = 'array' # object else: self.data['input_type'] = 'object' # input object field if field_data[2] not in self.inputs_set: self.data['inputs'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.inputs_set.add(field_data[2]) + self.current_json_obj = self.data['inputs'][-1] # Response type if (field_data[0] in self.response_json_array_roles or field_data[0] in self.response_json_object_roles): # array if field_data[0] in self.response_json_array_roles: self.data['return_type'] = 'array' # object else: self.data['return_type'] = 'object' # returned object field if field_data[2] not in self.returns_set: self.data['returns'].append({'name': field_data[2], 'type': field_data[1], 'doc': text}) self.returns_set.add(field_data[2]) + self.current_json_obj = self.data['returns'][-1] # Status Codes if field_data[0] in self.status_code_roles: if field_data[1] not in self.status_codes_set: self.data['status_codes'].append({'code': field_data[1], # noqa 'doc': text}) self.status_codes_set.add(field_data[1]) # Request Headers if field_data[0] in self.request_header_roles: if field_data[1] not in self.reqheaders_set: self.data['reqheaders'].append({'name': field_data[1], 'doc': text}) self.reqheaders_set.add(field_data[1]) # Response Headers if field_data[0] in self.response_header_roles: if field_data[1] not in self.resheaders_set: resheader = {'name': field_data[1], 'doc': text} self.data['resheaders'].append(resheader) self.resheaders_set.add(field_data[1]) if resheader['name'] == 'Content-Type' and \ resheader['doc'] == 'application/octet-stream': self.data['return_type'] = 'octet stream' def visit_paragraph(self, node): """ Visit relevant paragraphs to parse """ # only parsed top level paragraphs if isinstance(node.parent, docutils.nodes.block_quote): text = self.process_paragraph(str(node)) # endpoint description if (not text.startswith('**') and text not in self.data['description']): self.data['description'] += '\n\n' if self.data['description'] else '' # noqa self.data['description'] += text # http methods elif text.startswith('**Allowed HTTP Methods:**'): text = text.replace('**Allowed HTTP Methods:**', '') http_methods = text.strip().split(',') http_methods = [m[m.find('`')+1:-1].upper() for m in http_methods] self.data['urls'].append({'rule': self.urls[self.url_idx], 'methods': http_methods}) self.url_idx += 1 def visit_literal_block(self, node): """ Visit literal blocks """ text = node.astext() # literal block in endpoint description if not self.field_list_visited: self.data['description'] += \ ':\n\n%s\n' % textwrap.indent(text, '\t') # extract example url if ':swh_web_api:' in text: self.data['examples'].append( '/api/1/' + re.sub('.*`(.*)`.*', r'\1', text)) def visit_bullet_list(self, node): # bullet list in endpoint description if not self.field_list_visited: self.data['description'] += '\n\n' for child in node.traverse(): # process list item if isinstance(child, docutils.nodes.paragraph): line_text = self.process_paragraph(str(child)) self.data['description'] += '\t* %s\n' % line_text + elif self.current_json_obj: + self.current_json_obj['doc'] += '\n\n' + for child in node.traverse(): + # process list item + if isinstance(child, docutils.nodes.paragraph): + line_text = self.process_paragraph(str(child)) + self.current_json_obj['doc'] += '\t\t* %s\n' % line_text + self.current_json_obj = None def visit_warning(self, node): text = self.process_paragraph(str(node)) rst_warning = '\n\n.. warning::\n%s\n' % textwrap.indent(text, '\t') if rst_warning not in self.data['description']: self.data['description'] += rst_warning def unknown_visit(self, node): pass def depart_document(self, node): """ End of parsing extra processing """ default_methods = ['GET', 'HEAD', 'OPTIONS'] # ensure urls info is present and set default http methods if not self.data['urls']: for url in self.urls: self.data['urls'].append({'rule': url, 'methods': default_methods}) def unknown_departure(self, node): pass def _parse_httpdomain_doc(doc, data): doc_lines = doc.split('\n') doc_lines_filtered = [] urls = [] # httpdomain is a sphinx extension that is unknown to docutils but # fortunately we can still parse its directives' content, # so remove lines with httpdomain directives before executing the # rst parser from docutils for doc_line in doc_lines: if '.. http' not in doc_line: doc_lines_filtered.append(doc_line) else: url = doc_line[doc_line.find('/'):] # emphasize url arguments for html rendering url = re.sub(r'\((\w+)\)', r' **\(\1\)** ', url) urls.append(url) # parse the rst docstring and do not print system messages about # unknown httpdomain roles document = parse_rst('\n'.join(doc_lines_filtered), report_level=5) # remove the system_message nodes from the parsed document for node in document.traverse(docutils.nodes.system_message): node.parent.remove(node) # visit the document nodes to extract relevant endpoint info visitor = _HTTPDomainDocVisitor(document, urls, data) document.walkabout(visitor) class APIDocException(Exception): """ Custom exception to signal errors in the use of the APIDoc decorators """ def api_doc(route: str, noargs: bool = False, need_params: bool = False, tags: List[str] = [], handle_response: bool = False, api_version: str = '1'): """ Decorator for an API endpoint implementation used to generate a dedicated view displaying its HTML documentation. The documentation will be generated from the endpoint docstring based on sphinxcontrib-httpdomain format. Args: route: documentation page's route noargs: set to True if the route has no arguments, and its result should be displayed anytime its documentation is requested. Default to False need_params: specify the route requires query parameters otherwise errors will occur. It enables to avoid displaying the invalid response in its HTML documentation. Default to False. tags: Further information on api endpoints. Two values are possibly expected: * hidden: remove the entry points from the listing * upcoming: display the entry point but it is not followable handle_response: indicate if the decorated function takes care of creating the HTTP response or delegates that task to the apiresponse module api_version: api version string """ tags_set = set(tags) # @api_doc() Decorator call def decorator(f): # if the route is not hidden, add it to the index if 'hidden' not in tags_set: doc_data = get_doc_data(f, route, noargs) doc_desc = doc_data['description'] first_dot_pos = doc_desc.find('.') APIUrls.add_doc_route(route, doc_desc[:first_dot_pos+1], noargs=noargs, api_version=api_version, tags=tags_set) # create a dedicated view to display endpoint HTML doc @api_view(['GET', 'HEAD']) @wraps(f) def doc_view(request): doc_data = get_doc_data(f, route, noargs) return make_api_response(request, None, doc_data) route_name = '%s-doc' % route[1:-1].replace('/', '-') urlpattern = f'^{api_version}{route}doc/$' view_name = 'api-%s-%s' % (api_version, route_name) APIUrls.add_url_pattern(urlpattern, doc_view, view_name) @wraps(f) def documented_view(request, **kwargs): doc_data = get_doc_data(f, route, noargs) try: response = f(request, **kwargs) except Exception as exc: sentry_sdk.capture_exception(exc) if request.accepted_media_type == 'text/html' and \ need_params and not request.query_params: response = None else: return error_response(request, exc, doc_data) if handle_response: return response else: return make_api_response(request, response, doc_data) return documented_view return decorator @functools.lru_cache(maxsize=32) def get_doc_data(f, route, noargs): """ Build documentation data for the decorated api endpoint function """ data = { 'description': '', 'response_data': None, 'urls': [], 'args': [], 'params': [], 'input_type': '', 'inputs': [], 'resheaders': [], 'reqheaders': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [], 'route': route, 'noargs': noargs } if not f.__doc__: raise APIDocException('apidoc: expected a docstring' ' for function %s' % (f.__name__,)) # use raw docstring as endpoint documentation if sphinx # httpdomain is not used if '.. http' not in f.__doc__: data['description'] = f.__doc__ # else parse the sphinx httpdomain docstring with docutils # (except when building the swh-web documentation through autodoc # sphinx extension, not needed and raise errors with sphinx >= 1.7) elif 'SWH_WEB_DOC_BUILD' not in os.environ: _parse_httpdomain_doc(f.__doc__, data) # process input/returned object info for nicer html display inputs_list = '' returns_list = '' for inp in data['inputs']: # special case for array of non object type, for instance # :jsonarr string -: an array of string if ret['name'] != '-': returns_list += ('\t* **%s (%s)**: %s\n' % (ret['name'], ret['type'], ret['doc'])) data['inputs_list'] = inputs_list data['returns_list'] = returns_list return data DOC_COMMON_HEADERS = ''' :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request''' DOC_RESHEADER_LINK = ''' :resheader Link: indicates that a subsequent result page is available and contains the url pointing to it ''' DEFAULT_SUBSTITUTIONS = { 'common_headers': DOC_COMMON_HEADERS, 'resheader_link': DOC_RESHEADER_LINK, } def format_docstring(**substitutions): def decorator(f): f.__doc__ = f.__doc__.format(**{ **DEFAULT_SUBSTITUTIONS, **substitutions}) return f return decorator diff --git a/swh/web/assets/src/bundles/webapp/webapp.css b/swh/web/assets/src/bundles/webapp/webapp.css index a078c7ab..e0a96f85 100644 --- a/swh/web/assets/src/bundles/webapp/webapp.css +++ b/swh/web/assets/src/bundles/webapp/webapp.css @@ -1,609 +1,615 @@ /** * Copyright (C) 2018-2019 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ html { height: 100%; overflow-x: hidden; scroll-behavior: auto !important; } body { min-height: 100%; margin: 0; position: relative; padding-bottom: 120px; } a:active, a.active { outline: none; } code { background-color: #f9f2f4; } pre code { background-color: transparent; } footer { background-color: #262626; color: #fff; font-size: 0.8rem; position: absolute; bottom: 0; width: 100%; padding-top: 20px; padding-bottom: 20px; } footer a, footer a:visited, footer a:hover { color: #fecd1b; } footer a:hover { text-decoration: underline; } .link-color { color: #fecd1b; } pre { background-color: #f5f5f5; border: 1px solid #ccc; border-radius: 4px; padding: 9.5px; font-size: 0.8rem; } .btn.active { background-color: #e7e7e7; } .card { margin-bottom: 5px !important; overflow-x: auto; } .navbar-brand { padding: 5px; margin-right: 0; } .table { margin-bottom: 0; } .swh-table thead { background-color: #f2f4f5; border-top: 1px solid rgba(0, 0, 0, 0.2); font-weight: normal; } .swh-table-striped th { border-top: none; } .swh-table-striped tbody tr:nth-child(even) { background-color: #f2f4f5; } .swh-table-striped tbody tr:nth-child(odd) { background-color: #fff; } .swh-web-app-link a { text-decoration: none; border: none; } .swh-web-app-link:hover { background-color: #efeff2; } .table > thead > tr > th { border-top: none; border-bottom: 1px solid #e20026; } .table > tbody > tr > td { border-style: none; } .sitename .first-word, .sitename .second-word { color: rgba(0, 0, 0, 0.75); font-weight: normal; font-size: 1.2rem; } .sitename .first-word { font-family: 'Alegreya Sans', sans-serif; } .sitename .second-word { font-family: 'Alegreya', serif; } .swh-counter { font-size: 150%; } @media (max-width: 600px) { .swh-counter-container { margin-top: 1rem; } } .swh-http-error { margin: 0 auto; text-align: center; } .swh-http-error-head { color: #2d353c; font-size: 30px; } .swh-http-error-code { bottom: 60%; color: #2d353c; font-size: 96px; line-height: 80px; margin-bottom: 10px !important; } .swh-http-error-desc { font-size: 12px; color: #647788; text-align: center; } .swh-http-error-desc pre { display: inline-block; text-align: left; max-width: 800px; white-space: pre-wrap; } .popover { max-width: 97%; z-index: 40000; } .modal { text-align: center; padding: 0 !important; z-index: 50000; } .modal::before { content: ''; display: inline-block; height: 100%; vertical-align: middle; margin-right: -4px; } .modal-dialog { display: inline-block; text-align: left; vertical-align: middle; } .dropdown-submenu { position: relative; } .dropdown-submenu .dropdown-menu { top: 0; left: -100%; margin-top: -5px; margin-left: -2px; } .dropdown-item:hover, .dropdown-item:focus { background-color: rgba(0, 0, 0, 0.1); } a.dropdown-left::before { content: "\f0d9"; font-family: 'FontAwesome'; display: block; width: 20px; height: 20px; float: left; margin-left: 0; } #swh-navbar { border-top-style: none; border-left-style: none; border-right-style: none; border-bottom-style: solid; border-bottom-width: 5px; border-image: linear-gradient(to right, rgb(226, 0, 38) 0%, rgb(254, 205, 27) 100%) 1 1 1 1; width: 100%; padding: 5px; margin-bottom: 10px; margin-top: 30px; justify-content: normal; flex-wrap: nowrap; height: 72px; overflow: hidden; } #back-to-top { display: none; position: fixed; bottom: 30px; right: 30px; z-index: 10; } #back-to-top a img { display: block; width: 32px; height: 32px; background-size: 32px 32px; text-indent: -999px; overflow: hidden; } .swh-top-bar { direction: ltr; height: 30px; position: fixed; top: 0; left: 0; width: 100%; z-index: 99999; background-color: #262626; color: #fff; text-align: center; font-size: 14px; } .swh-top-bar ul { margin-top: 4px; padding-left: 0; white-space: nowrap; } .swh-top-bar li { display: inline-block; margin-left: 10px; margin-right: 10px; } .swh-top-bar a, .swh-top-bar a:visited { color: white; } .swh-top-bar a.swh-current-site, .swh-top-bar a.swh-current-site:visited { color: #fecd1b; } .swh-position-right { position: absolute; right: 0; } .swh-donate-link { border: 1px solid #fecd1b; background-color: #e20026; color: white !important; padding: 3px; border-radius: 3px; } .swh-navbar-content h4 { padding-top: 7px; } .swh-navbar-content .bread-crumbs { display: block; margin-left: -40px; } .swh-navbar-content .bread-crumbs li.bc-no-root { padding-top: 7px; } .main-sidebar { margin-top: 30px; } .content-wrapper { background: none; } .brand-image { max-height: 40px; } .brand-link { padding-top: 18.5px; padding-bottom: 18px; padding-left: 4px; border-bottom: 5px solid #e20026 !important; } .navbar-header a, ul.dropdown-menu a, ul.navbar-nav a, ul.nav-sidebar a { border-bottom-style: none; color: #323232; } .swh-sidebar .nav-link.active { color: #323232 !important; background-color: #e7e7e7 !important; } .swh-image-error { width: 80px; height: auto; } @media (max-width: 600px) { .card { min-width: 80%; } .swh-image-error { width: 40px; height: auto; } .swh-navbar-content h4 { font-size: 1rem; } .swh-donate-link { display: none; } } .form-check-label { padding-top: 4px; } .swh-id-option { display: inline-block; margin-right: 5px; line-height: 1rem; } .nav-pills .nav-link:not(.active):hover { color: rgba(0, 0, 0, 0.55); } .swh-heading-color { color: #e20026 !important; } .sidebar-mini.sidebar-collapse .main-sidebar:hover { width: 4.6rem; } .sidebar-mini.sidebar-collapse .main-sidebar:hover .user-panel > .info, .sidebar-mini.sidebar-collapse .main-sidebar:hover .nav-sidebar .nav-link p, .sidebar-mini.sidebar-collapse .main-sidebar:hover .brand-text { visibility: hidden !important; } .sidebar .nav-link p, .main-sidebar .brand-text, .sidebar .user-panel .info { transition: none; } .sidebar-mini.sidebar-mini.sidebar-collapse .sidebar { padding-right: 0; } .swh-words-logo { position: absolute; top: 0; left: 0; width: 73px; height: 73px; text-align: center; font-size: 10pt; color: rgba(0, 0, 0, 0.75); } .swh-words-logo:hover { text-decoration: none; } .swh-words-logo-swh { line-height: 1; padding-top: 13px; visibility: hidden; } hr.swh-faded-line { border: 0; height: 1px; background-image: linear-gradient(to left, #f0f0f0, #8c8b8b, #f0f0f0); } /* Ensure that section title with link is colored like standard section title */ .swh-readme h1 a, .swh-readme h2 a, .swh-readme h3 a, .swh-readme h4 a, .swh-readme h5 a, .swh-readme h6 a { color: #e20026; } /* Make list compact in reStructuredText rendering */ .swh-rst li p { margin-bottom: 0; } .swh-readme-txt pre { background: none; border: none; } .swh-coverage-col { padding-left: 10px; padding-right: 10px; } .swh-coverage { height: calc(65px + 1em); padding-top: 0.3rem; border: none; } .swh-coverage a { text-decoration: none; } .swh-coverage-logo { display: block; width: 100%; height: 50px; margin-left: auto; margin-right: auto; object-fit: contain; /* polyfill for old browsers, see https://github.com/bfred-it/object-fit-images */ font-family: 'object-fit: contain;'; } .swh-coverage-list { width: 100%; height: 320px; border: none; } tr.swh-tr-hover-highlight:hover td { background: #ededed; } tr.swh-api-doc-route a { text-decoration: none; } .swh-apidoc .col { margin: 10px; } +.swh-apidoc .swh-rst blockquote { + border: 0; + margin: 0; + padding: 0; +} + a.toggle-col { text-decoration: none; } a.toggle-col.col-hidden { text-decoration: line-through; } .admonition.warning { background: #fcf8e3; border: 1px solid #faebcc; padding: 15px; border-radius: 4px; } .admonition.warning p { margin-bottom: 0; } .admonition.warning .first { font-size: 1.5rem; } .swh-popover { max-height: 50vh; overflow-y: auto; overflow-x: auto; padding: 0; padding-right: 1.4em; } @media screen and (min-width: 768px) { .swh-popover { max-width: 50vw; } } .swh-metadata-table-row { border-top: 1px solid #ddd !important; } .swh-metadata-table-key { min-width: 200px; max-width: 200px; width: 200px; } .swh-metadata-table-value pre { white-space: pre-wrap; } .d3-wrapper { position: relative; height: 0; width: 100%; padding: 0; /* padding-bottom will be overwritten by JavaScript later */ padding-bottom: 100%; } .d3-wrapper > svg { position: absolute; height: 100%; width: 100%; left: 0; top: 0; } div.d3-tooltip { position: absolute; text-align: center; width: auto; height: auto; padding: 2px; font: 12px sans-serif; background: white; border: 1px solid black; border-radius: 4px; pointer-events: none; } .page-link { cursor: pointer; } .wrapper { overflow: hidden; } .swh-badge { padding-bottom: 1rem; cursor: pointer; } .swh-badge-html, .swh-badge-md, .swh-badge-rst { white-space: pre-wrap; } diff --git a/swh/web/common/middlewares.py b/swh/web/common/middlewares.py index cb6ca5ac..7fceaf59 100644 --- a/swh/web/common/middlewares.py +++ b/swh/web/common/middlewares.py @@ -1,71 +1,71 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -from bs4 import BeautifulSoup from htmlmin import minify import sentry_sdk +from swh.web.common.utils import prettify_html + class HtmlPrettifyMiddleware(object): """ Django middleware for prettifying generated HTML in development mode. """ def __init__(self, get_response): self.get_response = get_response def __call__(self, request): response = self.get_response(request) if 'text/html' in response.get('Content-Type', ''): if hasattr(response, 'content'): content = response.content - response.content = BeautifulSoup(content, 'lxml').prettify() + response.content = prettify_html(content) elif hasattr(response, 'streaming_content'): content = b''.join(response.streaming_content) - response.streaming_content = \ - BeautifulSoup(content, 'lxml').prettify() + response.streaming_content = prettify_html(content) return response class HtmlMinifyMiddleware(object): """ Django middleware for minifying generated HTML in production mode. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): response = self.get_response(request) if 'text/html' in response.get('Content-Type', ''): try: minified_html = minify(response.content.decode('utf-8')) response.content = minified_html.encode('utf-8') except Exception as exc: sentry_sdk.capture_exception(exc) return response class ThrottlingHeadersMiddleware(object): """ Django middleware for inserting rate limiting related headers in HTTP response. """ def __init__(self, get_response=None): self.get_response = get_response def __call__(self, request): resp = self.get_response(request) if 'RateLimit-Limit' in request.META: resp['X-RateLimit-Limit'] = request.META['RateLimit-Limit'] if 'RateLimit-Remaining' in request.META: resp['X-RateLimit-Remaining'] = request.META['RateLimit-Remaining'] if 'RateLimit-Reset' in request.META: resp['X-RateLimit-Reset'] = request.META['RateLimit-Reset'] return resp diff --git a/swh/web/common/swh_templatetags.py b/swh/web/common/swh_templatetags.py index 18e20c69..b2e9c0c4 100644 --- a/swh/web/common/swh_templatetags.py +++ b/swh/web/common/swh_templatetags.py @@ -1,160 +1,158 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import re -from inspect import cleandoc - from django import template from django.core.serializers.json import DjangoJSONEncoder from django.utils.safestring import mark_safe import sentry_sdk from swh.web.common.origin_save import get_savable_visit_types from swh.web.common.utils import rst_to_html register = template.Library() @register.filter -def safe_docstring_display(docstring): +def docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ - return rst_to_html(cleandoc(docstring)) + return rst_to_html(docstring) @register.filter def urlize_links_and_mails(text): """Utility function for decorating api links in browsable api. Args: text: whose content matching links should be transformed into contextual API or Browse html links. Returns The text transformed if any link is found. The text as is otherwise. """ try: if 'href="' not in text: text = re.sub(r'(http.*)', r'\1', text) return re.sub(r'([^ <>"]+@[^ <>"]+)', r'\1', text) except Exception as exc: sentry_sdk.capture_exception(exc) return text @register.filter def urlize_header_links(text): """Utility function for decorating headers links in browsable api. Args text: Text whose content contains Link header value Returns: The text transformed with html link if any link is found. The text as is otherwise. """ links = text.split(',') ret = '' for i, link in enumerate(links): ret += re.sub(r'<(http.*)>', r'<\1>', link) # add one link per line and align them if i != len(links) - 1: ret += '\n ' return ret @register.filter def jsonify(obj): """Utility function for converting a django template variable to JSON in order to use it in script tags. Args obj: Any django template context variable Returns: JSON representation of the variable. """ return mark_safe(json.dumps(obj, cls=DjangoJSONEncoder)) @register.filter def sub(value, arg): """Django template filter for subtracting two numbers Args: value (int/float): the value to subtract from arg (int/float): the value to subtract to Returns: int/float: The subtraction result """ return value - arg @register.filter def mul(value, arg): """Django template filter for multiplying two numbers Args: value (int/float): the value to multiply from arg (int/float): the value to multiply with Returns: int/float: The multiplication result """ return value * arg @register.filter def key_value(dict, key): """Django template filter to get a value in a dictionary. Args: dict (dict): a dictionary key (str): the key to lookup value Returns: The requested value in the dictionary """ return dict[key] @register.filter def visit_type_savable(visit_type): """Django template filter to check if a save request can be created for a given visit type. Args: visit_type (str): the type of visit Returns: If the visit type is saveable or not """ return visit_type in get_savable_visit_types() @register.filter def split(value, arg): """Django template filter to split a string. Args: value (str): the string to split arg (str): the split separator Returns: list: the split string parts """ return value.split(arg) diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py index 36781821..c0bfd76a 100644 --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -1,479 +1,494 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re from datetime import datetime, timezone from dateutil import parser as date_parser from dateutil import tz from typing import Optional, Dict, Any import docutils.parsers.rst import docutils.utils +from bs4 import BeautifulSoup + from docutils.core import publish_parts from docutils.writers.html5_polyglot import Writer, HTMLTranslator from django.urls import reverse as django_reverse from django.http import QueryDict, HttpRequest from prometheus_client.registry import CollectorRegistry from rest_framework.authentication import SessionAuthentication from swh.model.exceptions import ValidationError from swh.model.hashutil import hash_to_bytes from swh.model.identifiers import ( persistent_identifier, parse_persistent_identifier, CONTENT, DIRECTORY, ORIGIN, RELEASE, REVISION, SNAPSHOT ) from swh.web.common.exc import BadInputExc from swh.web.config import get_config SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) swh_object_icons = { 'branch': 'fa fa-code-fork', 'branches': 'fa fa-code-fork', 'content': 'fa fa-file-text', 'directory': 'fa fa-folder', 'person': 'fa fa-user', 'revisions history': 'fa fa-history', 'release': 'fa fa-tag', 'releases': 'fa fa-tag', 'revision': 'octicon-git-commit', 'snapshot': 'fa fa-camera', 'visits': 'fa fa-calendar', } def reverse(viewname: str, url_args: Optional[Dict[str, Any]] = None, query_params: Optional[Dict[str, Any]] = None, current_app: Optional[str] = None, urlconf: Optional[str] = None, request: Optional[HttpRequest] = None) -> str: """An override of django reverse function supporting query parameters. Args: viewname: the name of the django view from which to compute a url url_args: dictionary of url arguments indexed by their names query_params: dictionary of query parameters to append to the reversed url current_app: the name of the django app tighten to the view urlconf: url configuration module request: build an absolute URI if provided Returns: str: the url of the requested view with processed arguments and query parameters """ if url_args: url_args = {k: v for k, v in url_args.items() if v is not None} url = django_reverse(viewname, urlconf=urlconf, kwargs=url_args, current_app=current_app) if query_params: query_params = {k: v for k, v in query_params.items() if v} if query_params and len(query_params) > 0: query_dict = QueryDict('', mutable=True) for k in sorted(query_params.keys()): query_dict[k] = query_params[k] url += ('?' + query_dict.urlencode(safe='/;:')) if request is not None: url = request.build_absolute_uri(url) return url def datetime_to_utc(date): """Returns datetime in UTC without timezone info Args: date (datetime.datetime): input datetime with timezone info Returns: datetime.datetime: datetime in UTC without timezone info """ if date.tzinfo: return date.astimezone(tz.gettz('UTC')).replace(tzinfo=timezone.utc) else: return date def parse_timestamp(timestamp): """Given a time or timestamp (as string), parse the result as UTC datetime. Returns: datetime.datetime: a timezone-aware datetime representing the parsed value or None if the parsing fails. Samples: - 2016-01-12 - 2016-01-12T09:19:12+0100 - Today is January 1, 2047 at 8:21:00AM - 1452591542 """ if not timestamp: return None try: date = date_parser.parse(timestamp, ignoretz=False, fuzzy=True) return datetime_to_utc(date) except Exception: try: return datetime.utcfromtimestamp(float(timestamp)).replace( tzinfo=timezone.utc) except (ValueError, OverflowError) as e: raise BadInputExc(e) def shorten_path(path): """Shorten the given path: for each hash present, only return the first 8 characters followed by an ellipsis""" sha256_re = r'([0-9a-f]{8})[0-9a-z]{56}' sha1_re = r'([0-9a-f]{8})[0-9a-f]{32}' ret = re.sub(sha256_re, r'\1...', path) return re.sub(sha1_re, r'\1...', ret) def format_utc_iso_date(iso_date, fmt='%d %B %Y, %H:%M UTC'): """Turns a string representation of an ISO 8601 date string to UTC and format it into a more human readable one. For instance, from the following input string: '2017-05-04T13:27:13+02:00' the following one is returned: '04 May 2017, 11:27 UTC'. Custom format string may also be provided as parameter Args: iso_date (str): a string representation of an ISO 8601 date fmt (str): optional date formatting string Returns: str: a formatted string representation of the input iso date """ if not iso_date: return iso_date date = parse_timestamp(iso_date) return date.strftime(fmt) def gen_path_info(path): """Function to generate path data navigation for use with a breadcrumb in the swh web ui. For instance, from a path /folder1/folder2/folder3, it returns the following list:: [{'name': 'folder1', 'path': 'folder1'}, {'name': 'folder2', 'path': 'folder1/folder2'}, {'name': 'folder3', 'path': 'folder1/folder2/folder3'}] Args: path: a filesystem path Returns: list: a list of path data for navigation as illustrated above. """ path_info = [] if path: sub_paths = path.strip('/').split('/') path_from_root = '' for p in sub_paths: path_from_root += '/' + p path_info.append({'name': p, 'path': path_from_root.strip('/')}) return path_info def get_swh_persistent_id(object_type, object_id, scheme_version=1): """ Returns the persistent identifier for a swh object based on: * the object type * the object id * the swh identifiers scheme version Args: object_type (str): the swh object type (content/directory/release/revision/snapshot) object_id (str): the swh object id (hexadecimal representation of its hash value) scheme_version (int): the scheme version of the swh persistent identifiers Returns: str: the swh object persistent identifier Raises: BadInputExc: if the provided parameters do not enable to generate a valid identifier """ try: swh_id = persistent_identifier(object_type, object_id, scheme_version) except ValidationError as e: raise BadInputExc('Invalid object (%s) for swh persistent id. %s' % (object_id, e)) else: return swh_id def resolve_swh_persistent_id(swh_id, query_params=None): """ Try to resolve a Software Heritage persistent id into an url for browsing the pointed object. Args: swh_id (str): a Software Heritage persistent identifier query_params (django.http.QueryDict): optional dict filled with query parameters to append to the browse url Returns: dict: a dict with the following keys: * **swh_id_parsed (swh.model.identifiers.PersistentId)**: the parsed identifier * **browse_url (str)**: the url for browsing the pointed object """ swh_id_parsed = get_persistent_identifier(swh_id) object_type = swh_id_parsed.object_type object_id = swh_id_parsed.object_id browse_url = None query_dict = QueryDict('', mutable=True) if query_params and len(query_params) > 0: for k in sorted(query_params.keys()): query_dict[k] = query_params[k] if 'origin' in swh_id_parsed.metadata: query_dict['origin'] = swh_id_parsed.metadata['origin'] if object_type == CONTENT: query_string = 'sha1_git:' + object_id fragment = '' if 'lines' in swh_id_parsed.metadata: lines = swh_id_parsed.metadata['lines'].split('-') fragment += '#L' + lines[0] if len(lines) > 1: fragment += '-L' + lines[1] browse_url = reverse('browse-content', url_args={'query_string': query_string}, query_params=query_dict) + fragment elif object_type == DIRECTORY: browse_url = reverse('browse-directory', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == RELEASE: browse_url = reverse('browse-release', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == REVISION: browse_url = reverse('browse-revision', url_args={'sha1_git': object_id}, query_params=query_dict) elif object_type == SNAPSHOT: browse_url = reverse('browse-snapshot', url_args={'snapshot_id': object_id}, query_params=query_dict) elif object_type == ORIGIN: raise BadInputExc(('Origin PIDs (Persistent Identifiers) are not ' 'publicly resolvable because they are for ' 'internal usage only')) return {'swh_id_parsed': swh_id_parsed, 'browse_url': browse_url} def parse_rst(text, report_level=2): """ Parse a reStructuredText string with docutils. Args: text (str): string with reStructuredText markups in it report_level (int): level of docutils report messages to print (1 info 2 warning 3 error 4 severe 5 none) Returns: docutils.nodes.document: a parsed docutils document """ parser = docutils.parsers.rst.Parser() components = (docutils.parsers.rst.Parser,) settings = docutils.frontend.OptionParser( components=components).get_default_values() settings.report_level = report_level document = docutils.utils.new_document('rst-doc', settings=settings) parser.parse(text, document) return document def get_client_ip(request): """ Return the client IP address from an incoming HTTP request. Args: request (django.http.HttpRequest): the incoming HTTP request Returns: str: The client IP address """ x_forwarded_for = request.META.get('HTTP_X_FORWARDED_FOR') if x_forwarded_for: ip = x_forwarded_for.split(',')[0] else: ip = request.META.get('REMOTE_ADDR') return ip def context_processor(request): """ Django context processor used to inject variables in all swh-web templates. """ return { 'swh_object_icons': swh_object_icons, 'available_languages': None, 'swh_client_config': get_config()['client_config'], } class EnforceCSRFAuthentication(SessionAuthentication): """ Helper class to enforce CSRF validation on a DRF view when a user is not authenticated. """ def authenticate(self, request): user = getattr(request._request, 'user', None) self.enforce_csrf(request) return (user, None) def resolve_branch_alias(snapshot: Dict[str, Any], branch: Optional[Dict[str, Any]] ) -> Optional[Dict[str, Any]]: """ Resolve branch alias in snapshot content. Args: snapshot: a full snapshot content branch: a branch alias contained in the snapshot Returns: The real snapshot branch that got aliased. """ while branch and branch['target_type'] == 'alias': if branch['target'] in snapshot['branches']: branch = snapshot['branches'][branch['target']] else: from swh.web.common import service snp = service.lookup_snapshot( snapshot['id'], branches_from=branch['target'], branches_count=1) if snp and branch['target'] in snp['branches']: branch = snp['branches'][branch['target']] else: branch = None return branch def get_persistent_identifier(persistent_id): """Check if a persistent identifier is valid. Args: persistent_id: A string representing a Software Heritage persistent identifier. Raises: BadInputExc: if the provided persistent identifier can not be parsed. Return: A persistent identifier object. """ try: pid_object = parse_persistent_identifier(persistent_id) except ValidationError as ve: raise BadInputExc('Error when parsing identifier: %s' % ' '.join(ve.messages)) else: return pid_object def group_swh_persistent_identifiers(persistent_ids): """ Groups many Software Heritage persistent identifiers into a dictionary depending on their type. Args: persistent_ids (list): a list of Software Heritage persistent identifier objects Returns: A dictionary with: keys: persistent identifier types values: list(bytes) persistent identifiers id Raises: BadInputExc: if one of the provided persistent identifier can not be parsed. """ pids_by_type = { CONTENT: [], DIRECTORY: [], REVISION: [], RELEASE: [], SNAPSHOT: [] } for pid in persistent_ids: obj_id = pid.object_id obj_type = pid.object_type pids_by_type[obj_type].append(hash_to_bytes(obj_id)) return pids_by_type class _NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] _HTML_WRITER = Writer() _HTML_WRITER.translator_class = _NoHeaderHTMLTranslator def rst_to_html(rst: str) -> str: """ Convert reStructuredText document into HTML. Args: rst: A string containing a reStructuredText document Returns: Body content of the produced HTML conversion. """ settings = { 'initial_header_level': 2, } pp = publish_parts(rst, writer=_HTML_WRITER, settings_overrides=settings) return f'
{pp["html_body"]}
' + + +def prettify_html(html: str) -> str: + """ + Prettify an HTML document. + + Args: + html: Input HTML document + + Returns: + The prettified HTML document + """ + return BeautifulSoup(html, 'lxml').prettify() diff --git a/swh/web/templates/api/apidoc.html b/swh/web/templates/api/apidoc.html index c3a1966b..c088d19b 100644 --- a/swh/web/templates/api/apidoc.html +++ b/swh/web/templates/api/apidoc.html @@ -1,212 +1,212 @@ {% extends "layout.html" %} {% comment %} Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block title %}{{ heading }} – Software Heritage API {% endblock %} {% block navbar-content %} {% endblock %} {% block content %}
{% if description %}

Description

- {{ description | safe_docstring_display | safe }} + {{ description | docstring_display | safe }}
{% endif %} {% if response_data is not None %}

Request

{{ request.method }} {{ request.build_absolute_uri }}

Response

{% if status_code != 200 %}
Status Code
{{ status_code }}
{% endif %} {% if headers_data %}
Headers
{% for header_name, header_value in headers_data.items %} 
{{ header_name }} {{ header_value | urlize_header_links | safe }}
{% endfor %} {% endif %}
Body
{{ response_data | urlize_links_and_mails | safe }}
{% endif %}
{% if urls and urls|length > 0 %}
{% for url in urls %} - + {% endfor %}
URL Allowed Methods
{{ url.rule | safe_docstring_display | safe }}{{ url.rule | docstring_display | safe }} {{ url.methods | dictsort:0 | join:', ' }}
{% endif %} {% if args and args|length > 0 %}

Arguments

{% for arg in args %}
{{ arg.name }} ({{ arg.type }})
-
{{ arg.doc | safe_docstring_display | safe }}
+
{{ arg.doc | docstring_display | safe }}
{% endfor %}
{% endif %} {% if params and params|length > 0 %}

Query parameters

{% for param in params %}
{{ param.name }} ({{ param.type }})
-
{{ param.doc | safe_docstring_display | safe }}
+
{{ param.doc | docstring_display | safe }}
{% endfor %}
{% endif %} {% if reqheaders and reqheaders|length > 0 %}

Request headers

{% for header in reqheaders %}
{{ header.name }}
-
{{ header.doc | safe_docstring_display | safe }}
+
{{ header.doc | docstring_display | safe }}
{% endfor %}
{% endif %} {% if input_type %}

Request data

{{ input_type }}

{% if input_type == 'array' and inputs_list == '' %} {{ inputs.0.doc | safe }} {% elif input_type == 'array' and inputs_list != '' %} an array of objects containing the following keys: {% elif input_type == 'octet stream' %} raw data as an octet stream {% elif input_type == 'object' %} an object containing the following keys: {% endif %} {% if inputs_list != '' %} - {{ inputs_list | safe_docstring_display | safe }} + {{ inputs_list | docstring_display | safe }} {% endif %}

{% endif %} {% if resheaders and resheaders|length > 0 %}

Response headers

{% for header in resheaders %}
{{ header.name }}
-
{{ header.doc | safe_docstring_display | safe }}
+
{{ header.doc | docstring_display | safe }}
{% endfor %}
{% endif %} {% if return_type %}

Returns

{{ return_type }}

{% if return_type == 'array' and returns_list == '' %} {{ returns.0.doc | safe }} {% elif return_type == 'array' and returns_list != '' %} an array of objects containing the following keys: {% elif return_type == 'octet stream' %} raw data as an octet stream {% elif return_type == 'object' %} an object containing the following keys: {% endif %} {% if returns_list != '' %} - {{ returns_list | safe_docstring_display | safe }} + {{ returns_list | docstring_display | safe }} {% endif %}

{% endif %} {% if status_codes and status_codes|length > 0 %}

HTTP status codes

{% for status in status_codes %}
{{ status.code }}
-
{{ status.doc | safe_docstring_display | safe }}
+
{{ status.doc | docstring_display | safe }}
{% endfor %}
{% endif %} {% if examples and examples|length > 0 %}

Examples

{% for example in examples %}
{{ example }}
{% endfor %}
{% endif %}
{% endblock %} diff --git a/swh/web/templates/api/endpoints.html b/swh/web/templates/api/endpoints.html index 71da9c62..6c94d5d7 100644 --- a/swh/web/templates/api/endpoints.html +++ b/swh/web/templates/api/endpoints.html @@ -1,82 +1,82 @@ {% extends "layout.html" %} {% comment %} Copyright (C) 2015-2019 The Software Heritage developers See the AUTHORS file at the top-level directory of this distribution License: GNU Affero General Public License version 3, or any later version See top-level LICENSE file for more information {% endcomment %} {% load swh_templatetags %} {% block title %} Endpoints – Software Heritage API {% endblock %} {% block navbar-content %}
{% endblock %} {% block content %}

Below you can find a list of the available endpoints for version 1 of the Software Heritage API. For a more general introduction please refer to the API overview.

Endpoints marked "available" are considered stable for the current version of the API; endpoints marked "upcoming" are work in progress that will be stabilized in the near future.

{% for route, doc in doc_routes %} {% if doc.tags|length > 0 %} {% else %} {% endif %} {% endfor %}
Endpoint Description
{% url doc.route_view_name %} {{ doc.route }} - {{ doc.doc_intro | safe_docstring_display | safe }} + {{ doc.doc_intro | docstring_display | safe }}
{% endblock %} diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py index c45f220a..5f907e6f 100644 --- a/swh/web/tests/api/test_apidoc.py +++ b/swh/web/tests/api/test_apidoc.py @@ -1,431 +1,454 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import textwrap + import pytest from rest_framework.response import Response from swh.storage.exc import StorageDBError, StorageAPIError from swh.web.api.apidoc import api_doc, _parse_httpdomain_doc from swh.web.api.apiurls import api_route from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc -from swh.web.common.utils import reverse -from swh.web.tests.django_asserts import assert_template_used, assert_contains +from swh.web.common.utils import reverse, prettify_html +from swh.web.tests.django_asserts import assert_template_used _httpdomain_doc = """ .. http:get:: /api/1/revision/(sha1_git)/ Get information about a revision in the archive. Revisions are identified by **sha1** checksums, compatible with Git commit identifiers. See :func:`swh.model.identifiers.revision_identifier` in our data model module for details about how they are computed. :param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier :reqheader Accept: the requested response content type, either ``application/json`` (default) or ``application/yaml`` :resheader Content-Type: this depends on :http:header:`Accept` header of request :json object author: information about the author of the revision :>json object committer: information about the committer of the revision :>json string committer_date: ISO representation of the commit date (in UTC) :>json string date: ISO representation of the revision date (in UTC) :>json string directory: the unique identifier that revision points to :>json string directory_url: link to :http:get:`/api/1/directory/(sha1_git)/[(path)/]` to get information about the directory associated to the revision :>json string id: the revision unique identifier :>json boolean merge: whether or not the revision corresponds to a merge commit :>json string message: the message associated to the revision :>json array parents: the parents of the revision, i.e. the previous revisions that head directly to it, each entry of that array contains an unique parent revision identifier but also a link to :http:get:`/api/1/revision/(sha1_git)/` to get more information about it :>json string type: the type of the revision **Allowed HTTP Methods:** :http:method:`get`, :http:method:`head` :statuscode 200: no error :statuscode 400: an invalid **sha1_git** value has been provided :statuscode 404: requested revision can not be found in the archive **Request:** .. parsed-literal:: :swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/` """ _exception_http_code = { BadInputExc: 400, ForbiddenExc: 403, NotFoundExc: 404, Exception: 500, StorageAPIError: 503, StorageDBError: 503, } def test_apidoc_nodoc_failure(): with pytest.raises(Exception): @api_doc('/my/nodoc/url/') def apidoc_nodoc_tester(request, arga=0, argb=0): return Response(arga + argb) @api_route(r'/some/(?P[0-9]+)/(?P[0-9]+)/', 'api-1-some-doc-route') @api_doc('/some/doc/route/') def apidoc_route(request, myarg, myotherarg, akw=0): """ Sample doc """ return {'result': int(myarg) + int(myotherarg) + akw} def test_apidoc_route_doc(client): url = reverse('api-1-some-doc-route-doc') rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') def test_apidoc_route_fn(api_client): url = reverse('api-1-some-doc-route', url_args={'myarg': 1, 'myotherarg': 1}) rv = api_client.get(url) assert rv.status_code == 200, rv.data @api_route(r'/test/error/(?P.+)/', 'api-1-test-error') @api_doc('/test/error/') def apidoc_test_error_route(request, exc_name): """ Sample doc """ for e in _exception_http_code.keys(): if e.__name__ == exc_name: raise e('Error') def test_apidoc_error(api_client): for exc, code in _exception_http_code.items(): url = reverse('api-1-test-error', url_args={'exc_name': exc.__name__}) rv = api_client.get(url) assert rv.status_code == code, rv.data @api_route(r'/some/full/(?P[0-9]+)/(?P[0-9]+)/', 'api-1-some-complete-doc-route') @api_doc('/some/complete/doc/route/') def apidoc_full_stack(request, myarg, myotherarg, akw=0): """ Sample doc """ return {'result': int(myarg) + int(myotherarg) + akw} def test_apidoc_full_stack_doc(client): url = reverse('api-1-some-complete-doc-route-doc') rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') def test_apidoc_full_stack_fn(api_client): url = reverse('api-1-some-complete-doc-route', url_args={'myarg': 1, 'myotherarg': 1}) rv = api_client.get(url) assert rv.status_code == 200, rv.data @api_route(r'/test/post/only/', 'api-1-test-post-only', methods=['POST']) @api_doc('/test/post/only/') def apidoc_test_post_only(request, exc_name): """ Sample doc """ return {'result': 'some data'} def test_apidoc_post_only(client): # a dedicated view accepting GET requests should have # been created to display the HTML documentation url = reverse('api-1-test-post-only-doc') rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') def test_api_doc_parse_httpdomain(): doc_data = { 'description': '', 'urls': [], 'args': [], 'params': [], 'resheaders': [], 'reqheaders': [], 'input_type': '', 'inputs': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [] } _parse_httpdomain_doc(_httpdomain_doc, doc_data) expected_urls = [{ 'rule': '/api/1/revision/ **\\(sha1_git\\)** /', 'methods': ['GET', 'HEAD'] }] assert 'urls' in doc_data assert doc_data['urls'] == expected_urls expected_description = ('Get information about a revision in the archive. ' 'Revisions are identified by **sha1** checksums, ' 'compatible with Git commit identifiers. See ' '**swh.model.identifiers.revision_identifier** in ' 'our data model module for details about how they ' 'are computed.') assert 'description' in doc_data assert doc_data['description'] == expected_description expected_args = [{ 'name': 'sha1_git', 'type': 'string', 'doc': ('hexadecimal representation of the revision ' '**sha1_git** identifier') }] assert 'args' in doc_data assert doc_data['args'] == expected_args expected_params = [] assert 'params' in doc_data assert doc_data['params'] == expected_params expected_reqheaders = [{ 'doc': ('the requested response content type, either ' - '``application/json`` or ``application/yaml``'), + '``application/json`` (default) or ``application/yaml``'), 'name': 'Accept' }] assert 'reqheaders' in doc_data assert doc_data['reqheaders'] == expected_reqheaders expected_resheaders = [{ 'doc': 'this depends on **Accept** header of request', 'name': 'Content-Type' }] assert 'resheaders' in doc_data assert doc_data['resheaders'] == expected_resheaders expected_statuscodes = [ { 'code': '200', 'doc': 'no error' }, { 'code': '400', 'doc': 'an invalid **sha1_git** value has been provided' }, { 'code': '404', 'doc': 'requested revision can not be found in the archive' } ] assert 'status_codes' in doc_data assert doc_data['status_codes'] == expected_statuscodes expected_input_type = 'object' assert 'input_type' in doc_data assert doc_data['input_type'] == expected_input_type expected_inputs = [ { 'name': 'n', 'type': 'int', 'doc': 'sample input integer' }, { 'name': 's', 'type': 'string', 'doc': 'sample input string' }, { 'name': 'a', 'type': 'array', 'doc': 'sample input array' }, ] assert 'inputs' in doc_data assert doc_data['inputs'] == expected_inputs expected_return_type = 'object' assert 'return_type' in doc_data assert doc_data['return_type'] == expected_return_type expected_returns = [ { 'name': 'author', 'type': 'object', 'doc': 'information about the author of the revision' }, { 'name': 'committer', 'type': 'object', 'doc': 'information about the committer of the revision' }, { 'name': 'committer_date', 'type': 'string', 'doc': 'ISO representation of the commit date (in UTC)' }, { 'name': 'date', 'type': 'string', 'doc': 'ISO representation of the revision date (in UTC)' }, { 'name': 'directory', 'type': 'string', 'doc': 'the unique identifier that revision points to' }, { 'name': 'directory_url', 'type': 'string', 'doc': ('link to ``_ to get information about ' 'the directory associated to the revision') }, { 'name': 'id', 'type': 'string', 'doc': 'the revision unique identifier' }, { 'name': 'merge', 'type': 'boolean', 'doc': 'whether or not the revision corresponds to a merge commit' }, { 'name': 'message', 'type': 'string', 'doc': 'the message associated to the revision' }, { 'name': 'parents', 'type': 'array', 'doc': ('the parents of the revision, i.e. the previous revisions ' 'that head directly to it, each entry of that array ' 'contains an unique parent revision identifier but also a ' 'link to ``_ to get more information ' 'about it') }, { 'name': 'type', 'type': 'string', 'doc': 'the type of the revision' } ] assert 'returns' in doc_data assert doc_data['returns'] == expected_returns expected_examples = [ '/api/1/revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/' ] assert 'examples' in doc_data assert doc_data['examples'] == expected_examples @api_route(r'/post/endpoint/', 'api-1-post-endpoint', methods=['POST']) @api_doc('/post/endpoint/') def apidoc_test_post_endpoint(request): """ .. http:post:: /api/1/post/endpoint/ Endpoint documentation :jsonarr string type: swh object type - :>jsonarr string sha1_git: swh object sha1_git - :>jsonarr boolean found: whether the object was found or not + :>json object : an object whose keys are input persistent + identifiers and values objects with the following keys: + + * **known (bool)**: whether the object was found """ pass def test_apidoc_input_output_doc(client): url = reverse('api-1-post-endpoint-doc') rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') - input_html_doc = ( - '
\n' - '
array
\n' - '
\n' - '

\n' - ' \n' - ' Input array of pids\n' - ' \n' - ' \n' - '

\n' - '
\n' - '
\n' - ) - - output_html_doc = ( - '
\n' - '
array
\n' - '
\n' - '

\n' - ' \n' - ' an array of objects containing the following keys:\n' - ' \n' - ' \n' - '

    \n' - '

  • type (string): swh object type

    • \n' - '

  • sha1_git (string): swh object sha1_git

    • \n' # noqa - '

  • found (boolean): whether the object was found or not

    • \n' # noqa - '
\n' - '
\n' - ' \n' - '

\n' - '
\n' - '
' - ) - - assert_contains(rv, input_html_doc) - assert_contains(rv, output_html_doc) + input_html_doc = textwrap.indent(( + '
\n' + '
\n' + ' array\n' + '
\n' + '
\n' + '

\n' + ' Input array of pids\n' + '

\n' + '
\n' + '
\n' + ), ' '*7) + + output_html_doc = textwrap.indent(( + '
\n' + '
\n' + ' object\n' + '
\n' + '
\n' + '

\n' + ' an object containing the following keys:\n' + '

\n' + '
\n' + '
\n' + '
    \n' + '
  • \n' + '

    \n' + ' \n' + ' <swh_pid> (object)\n' + ' \n' + ' : an object whose keys are input persistent identifiers' + ' and values objects with the following keys:\n' + '

    \n' + '
    \n' + '
      \n' + '
    • \n' + '

      \n' + ' \n' + ' known (bool)\n' + ' \n' + ' : whether the object was found\n' + '

      \n' + '
      • \n' + '
    \n' + '
    \n' + '
    • \n' + '
\n' + '
\n' + '
\n' + '
\n' + '
\n' + ), ' '*7) + + html = prettify_html(rv.content) + + assert input_html_doc in html + assert output_html_doc in html diff --git a/swh/web/tests/common/test_templatetags.py b/swh/web/tests/common/test_templatetags.py index d4076eb9..93e4d134 100644 --- a/swh/web/tests/common/test_templatetags.py +++ b/swh/web/tests/common/test_templatetags.py @@ -1,61 +1,63 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.web.common.swh_templatetags import ( - urlize_links_and_mails, urlize_header_links, safe_docstring_display + urlize_links_and_mails, urlize_header_links, docstring_display ) def test_urlize_http_link(): link = 'https://example.com/api/1/abc/' expected_content = f'{link}' assert urlize_links_and_mails(link) == expected_content def test_urlize_email(): email = 'someone@example.com' expected_content = f'{email}' assert urlize_links_and_mails(email) == expected_content def test_urlize_header_links(): next_link = 'https://example.com/api/1/abc/' prev_link = 'https://example.com/api/1/def/' content = f'<{next_link}>; rel="next"\n<{prev_link}>; rel="prev"' expected_content = ( f'<{next_link}>; rel="next"\n' f'<{prev_link}>; rel="prev"') assert urlize_header_links(content) == expected_content -def test_safe_docstring_display(): +def test_docstring_display(): # update api link with html links content with links docstring = ( 'This is my list header:\n\n' ' - Here is item 1, with a continuation\n' ' line right here\n' ' - Here is item 2\n\n' ' Here is something that is not part of the list' ) expected_docstring = ( '
' '

This is my list header:

\n' + '
\n' '
    \n' '

  • Here is item 1, with a continuation\n' 'line right here

    • \n' '

  • Here is item 2

    • \n' '
\n' '

Here is something that is not part of the list

\n' + '
\n' '
' ) - assert safe_docstring_display(docstring) == expected_docstring + assert docstring_display(docstring) == expected_docstring