diff --git a/swh/web/api/apidoc.py b/swh/web/api/apidoc.py --- a/swh/web/api/apidoc.py +++ b/swh/web/api/apidoc.py @@ -3,14 +3,17 @@ # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information -import docutils.nodes -import docutils.parsers.rst -import docutils.utils + import functools from functools import wraps import os import re import textwrap +from typing import List + +import docutils.nodes +import docutils.parsers.rst +import docutils.utils from rest_framework.decorators import api_view import sentry_sdk @@ -31,6 +34,10 @@ # httpdomain roles we want to parse (based on sphinxcontrib.httpdomain 1.6) parameter_roles = ('param', 'parameter', 'arg', 'argument') + request_json_object_roles = ('reqjsonobj', 'reqjson', 'jsonobj', '>json') response_json_array_roles = ('resjsonarr', '>jsonarr') @@ -50,6 +57,7 @@ self.data = data self.args_set = set() self.params_set = set() + self.inputs_set = set() self.returns_set = set() self.status_codes_set = set() self.reqheaders_set = set() @@ -112,9 +120,24 @@ 'type': field_data[1], 'doc': text}) self.params_set.add(field_data[2]) + # Request data type + if (field_data[0] in self.request_json_array_roles or + field_data[0] in self.request_json_object_roles): + # array + if field_data[0] in self.request_json_array_roles: + self.data['input_type'] = 'array' + # object + else: + self.data['input_type'] = 'object' + # input object field + if field_data[2] not in self.inputs_set: + self.data['inputs'].append({'name': field_data[2], + 'type': field_data[1], + 'doc': text}) + self.inputs_set.add(field_data[2]) # Response type - if field_data[0] in self.response_json_array_roles or \ - field_data[0] in self.response_json_object_roles: + if (field_data[0] in self.response_json_array_roles or + field_data[0] in self.response_json_object_roles): # array if field_data[0] in self.response_json_array_roles: self.data['return_type'] = 'array' @@ -253,59 +276,61 @@ """ -def api_doc(route, noargs=False, need_params=False, tags=[], - handle_response=False, api_version='1'): +def api_doc(route: str, noargs: bool = False, need_params: bool = False, + tags: List[str] = [], handle_response: bool = False, + api_version: str = '1'): """ - Decorate an API function to register it in the API doc route index - and create the corresponding DRF route. + Decorator for an API endpoint implementation used to generate a dedicated + view displaying its HTML documentation. + + The documentation will be generated from the endpoint docstring based on + sphinxcontrib-httpdomain format. Args: - route (str): documentation page's route - noargs (boolean): set to True if the route has no arguments, and its + route: documentation page's route + noargs: set to True if the route has no arguments, and its result should be displayed anytime its documentation is requested. Default to False - need_params (boolean): specify the route requires query parameters + need_params: specify the route requires query parameters otherwise errors will occur. It enables to avoid displaying the invalid response in its HTML documentation. Default to False. - tags (list): Further information on api endpoints. Two values are + tags: Further information on api endpoints. Two values are possibly expected: * hidden: remove the entry points from the listing * upcoming: display the entry point but it is not followable - handle_response (boolean): indicate if the decorated function takes + handle_response: indicate if the decorated function takes care of creating the HTTP response or delegates that task to the apiresponse module - api_version (str): api version string - + api_version: api version string """ - urlpattern = '^' + api_version + route + '$' - tags = set(tags) + + tags_set = set(tags) # @api_doc() Decorator call def decorator(f): - - # If the route is not hidden, add it to the index - if 'hidden' not in tags: + # if the route is not hidden, add it to the index + if 'hidden' not in tags_set: doc_data = get_doc_data(f, route, noargs) doc_desc = doc_data['description'] first_dot_pos = doc_desc.find('.') - APIUrls.add_route(route, doc_desc[:first_dot_pos+1], - tags=tags) + APIUrls.add_doc_route(route, doc_desc[:first_dot_pos+1], + noargs=noargs, api_version=api_version, + tags=tags_set) - # If the decorated route has arguments, we create a specific - # documentation view - if not noargs: + # create a dedicated view to display endpoint HTML doc + @api_view(['GET', 'HEAD']) + @wraps(f) + def doc_view(request): + doc_data = get_doc_data(f, route, noargs) + return make_api_response(request, None, doc_data) - @api_view(['GET', 'HEAD']) - @wraps(f) - def doc_view(request): - doc_data = get_doc_data(f, route, noargs) - return make_api_response(request, None, doc_data) + route_name = '%s-doc' % route[1:-1].replace('/', '-') + urlpattern = f'^{api_version}{route}doc/$' - view_name = 'api-%s-%s' % \ - (api_version, route[1:-1].replace('/', '-')) - APIUrls.add_url_pattern(urlpattern, doc_view, view_name) + view_name = 'api-%s-%s' % (api_version, route_name) + APIUrls.add_url_pattern(urlpattern, doc_view, view_name) @wraps(f) def documented_view(request, **kwargs): @@ -342,6 +367,8 @@ 'urls': [], 'args': [], 'params': [], + 'input_type': '', + 'inputs': [], 'resheaders': [], 'reqheaders': [], 'return_type': '', @@ -366,11 +393,22 @@ # sphinx extension, not needed and raise errors with sphinx >= 1.7) elif 'SWH_WEB_DOC_BUILD' not in os.environ: _parse_httpdomain_doc(f.__doc__, data) - # process returned object info for nicer html display + # process input/returned object info for nicer html display + inputs_list = '' returns_list = '' + for inp in data['inputs']: + # special case for array of non object type, for instance + # :jsonarr string -: an array of string + if ret['name'] != '-': + returns_list += ('\t* **%s (%s)**: %s\n' % + (ret['name'], ret['type'], ret['doc'])) + data['inputs_list'] = inputs_list data['returns_list'] = returns_list return data diff --git a/swh/web/api/apiresponse.py b/swh/web/api/apiresponse.py --- a/swh/web/api/apiresponse.py +++ b/swh/web/api/apiresponse.py @@ -135,8 +135,13 @@ doc_env['response_data'] = data doc_env['heading'] = shorten_path(str(request.path)) + # generate breadcrumbs data if 'route' in doc_env: doc_env['endpoint_path'] = gen_path_info(doc_env['route']) + for i in range(len(doc_env['endpoint_path']) - 1): + doc_env['endpoint_path'][i]['path'] += '/doc/' + if not doc_env['noargs']: + doc_env['endpoint_path'][-1]['path'] += '/doc/' response_args['data'] = doc_env response_args['template_name'] = 'api/apidoc.html' diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -30,13 +30,18 @@ return cls._apidoc_routes @classmethod - def add_route(cls, route, docstring, **kwargs): + def add_doc_route(cls, route, docstring, noargs=False, + api_version='1', **kwargs): """ Add a route to the self-documenting API reference """ - route_view_name = 'api-1-%s' % route[1:-1].replace('/', '-') + route_name = route[1:-1].replace('/', '-') + if not noargs: + route_name = '%s-doc' % route_name + route_view_name = 'api-%s-%s' % (api_version, route_name) if route not in cls._apidoc_routes: d = {'docstring': docstring, + 'route': '/api/%s%s' % (api_version, route), 'route_view_name': route_view_name} for k, v in kwargs.items(): d[k] = v diff --git a/swh/web/api/views/identifiers.py b/swh/web/api/views/identifiers.py --- a/swh/web/api/views/identifiers.py +++ b/swh/web/api/views/identifiers.py @@ -68,7 +68,7 @@ @api_route(r'/known/', 'api-1-swh-pid-known', methods=['POST']) -@api_doc('/known/', noargs=True, tags=['hidden']) +@api_doc('/known/', tags=['hidden']) @format_docstring() def api_swh_pid_known(request): """ diff --git a/swh/web/templates/api/apidoc.html b/swh/web/templates/api/apidoc.html --- a/swh/web/templates/api/apidoc.html +++ b/swh/web/templates/api/apidoc.html @@ -19,7 +19,7 @@
  • endpoints
    • {% for endpoint in endpoint_path %}
    • - {% if endpoint.name != 'stat' and endpoint.name != 'vault' and endpoint.path != 'vault/revision' %} + {% if endpoint.name != 'stat' and endpoint.name != 'vault' and endpoint.path != 'vault/revision/doc/' %}
  • {{ endpoint.name }}
    • {% else %}
  • {{ endpoint.name }}
    • @@ -116,6 +116,31 @@
    {% endif %} +{% if input_type %} +
    +

    Request data

    +
    +
    {{ input_type }}
    +
    +

    + {% if input_type == 'array' and inputs_list == '' %} + {{ inputs.0.doc | safe }} + {% elif input_type == 'array' and inputs_list != '' %} + an array of objects containing the following keys: + {% elif input_type == 'octet stream' %} + raw data as an octet stream + {% elif input_type == 'object' %} + an object containing the following keys: + {% endif %} + {% if inputs_list != '' %} + {{ inputs_list | safe_docstring_display | safe }} + {% endif %} +

    +
    +
    +
    +
    +{% endif %} {% if resheaders and resheaders|length > 0 %}

    Response headers

    @@ -135,14 +160,18 @@
    {{ return_type }}

    - {% if return_type == 'array' %} + {% if return_type == 'array' and returns_list == '' %} + {{ returns.0.doc | safe }} + {% elif return_type == 'array' and returns_list != '' %} an array of objects containing the following keys: {% elif return_type == 'octet stream' %} - the raw data as an octet stream - {% else %} + raw data as an octet stream + {% elif return_type == 'object' %} an object containing the following keys: {% endif %} - {{ returns_list | safe_docstring_display | safe }} + {% if returns_list != '' %} + {{ returns_list | safe_docstring_display | safe }} + {% endif %}

    diff --git a/swh/web/templates/api/endpoints.html b/swh/web/templates/api/endpoints.html --- a/swh/web/templates/api/endpoints.html +++ b/swh/web/templates/api/endpoints.html @@ -53,7 +53,7 @@ {% else %} - {% url doc.route_view_name %} + {{ doc.route }} {% endif %} diff --git a/swh/web/tests/api/test_apidoc.py b/swh/web/tests/api/test_apidoc.py --- a/swh/web/tests/api/test_apidoc.py +++ b/swh/web/tests/api/test_apidoc.py @@ -12,10 +12,11 @@ from swh.web.api.apidoc import api_doc, _parse_httpdomain_doc from swh.web.api.apiurls import api_route from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc -from swh.web.tests.django_asserts import assert_template_used +from swh.web.common.utils import reverse +from swh.web.tests.django_asserts import assert_template_used, assert_contains -httpdomain_doc = """ +_httpdomain_doc = """ .. http:get:: /api/1/revision/(sha1_git)/ Get information about a revision in the archive. @@ -32,6 +33,10 @@ :resheader Content-Type: this depends on :http:header:`Accept` header of request + :json object author: information about the author of the revision :>json object committer: information about the committer of the revision :>json string committer_date: ISO representation of the commit date @@ -66,7 +71,7 @@ """ -exception_http_code = { +_exception_http_code = { BadInputExc: 400, ForbiddenExc: 403, NotFoundExc: 404, @@ -84,7 +89,7 @@ @api_route(r'/some/(?P[0-9]+)/(?P[0-9]+)/', - 'some-doc-route') + 'api-1-some-doc-route') @api_doc('/some/doc/route/') def apidoc_route(request, myarg, myotherarg, akw=0): """ @@ -92,42 +97,44 @@ """ return {'result': int(myarg) + int(myotherarg) + akw} -# remove deprecation warnings related to docutils -@pytest.mark.filterwarnings( - 'ignore:.*U.*mode is deprecated:DeprecationWarning') + def test_apidoc_route_doc(client): - rv = client.get('/api/1/some/doc/route/', HTTP_ACCEPT='text/html') + url = reverse('api-1-some-doc-route-doc') + rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') def test_apidoc_route_fn(api_client): - rv = api_client.get('/api/1/some/1/1/') - + url = reverse('api-1-some-doc-route', + url_args={'myarg': 1, 'myotherarg': 1}) + rv = api_client.get(url) assert rv.status_code == 200, rv.data -@api_route(r'/test/error/(?P.+)/', 'test-error') +@api_route(r'/test/error/(?P.+)/', 'api-1-test-error') @api_doc('/test/error/') def apidoc_test_error_route(request, exc_name): """ Sample doc """ - for e in exception_http_code.keys(): + for e in _exception_http_code.keys(): if e.__name__ == exc_name: raise e('Error') def test_apidoc_error(api_client): - for exc, code in exception_http_code.items(): - rv = api_client.get('/api/1/test/error/%s/' % exc.__name__) + for exc, code in _exception_http_code.items(): + url = reverse('api-1-test-error', + url_args={'exc_name': exc.__name__}) + rv = api_client.get(url) assert rv.status_code == code, rv.data @api_route(r'/some/full/(?P[0-9]+)/(?P[0-9]+)/', - 'some-complete-doc-route') + 'api-1-some-complete-doc-route') @api_doc('/some/complete/doc/route/') def apidoc_full_stack(request, myarg, myotherarg, akw=0): """ @@ -136,21 +143,40 @@ return {'result': int(myarg) + int(myotherarg) + akw} -# remove deprecation warnings related to docutils -@pytest.mark.filterwarnings( - 'ignore:.*U.*mode is deprecated:DeprecationWarning') def test_apidoc_full_stack_doc(client): - rv = client.get('/api/1/some/complete/doc/route/', HTTP_ACCEPT='text/html') + url = reverse('api-1-some-complete-doc-route-doc') + rv = client.get(url, HTTP_ACCEPT='text/html') assert rv.status_code == 200, rv.content assert_template_used(rv, 'api/apidoc.html') def test_apidoc_full_stack_fn(api_client): - rv = api_client.get('/api/1/some/full/1/1/') + url = reverse('api-1-some-complete-doc-route', + url_args={'myarg': 1, 'myotherarg': 1}) + rv = api_client.get(url) assert rv.status_code == 200, rv.data +@api_route(r'/test/post/only/', 'api-1-test-post-only', + methods=['POST']) +@api_doc('/test/post/only/') +def apidoc_test_post_only(request, exc_name): + """ + Sample doc + """ + return {'result': 'some data'} + + +def test_apidoc_post_only(client): + # a dedicated view accepting GET requests should have + # been created to display the HTML documentation + url = reverse('api-1-test-post-only-doc') + rv = client.get(url, HTTP_ACCEPT='text/html') + assert rv.status_code == 200, rv.content + assert_template_used(rv, 'api/apidoc.html') + + def test_api_doc_parse_httpdomain(): doc_data = { 'description': '', @@ -159,13 +185,15 @@ 'params': [], 'resheaders': [], 'reqheaders': [], + 'input_type': '', + 'inputs': [], 'return_type': '', 'returns': [], 'status_codes': [], 'examples': [] } - _parse_httpdomain_doc(httpdomain_doc, doc_data) + _parse_httpdomain_doc(_httpdomain_doc, doc_data) expected_urls = [{ 'rule': '/api/1/revision/ **\\(sha1_git\\)** /', @@ -234,10 +262,36 @@ assert 'status_codes' in doc_data assert doc_data['status_codes'] == expected_statuscodes + expected_input_type = 'object' + + assert 'input_type' in doc_data + assert doc_data['input_type'] == expected_input_type + + expected_inputs = [ + { + 'name': 'n', + 'type': 'int', + 'doc': 'sample input integer' + }, + { + 'name': 's', + 'type': 'string', + 'doc': 'sample input string' + }, + { + 'name': 'a', + 'type': 'array', + 'doc': 'sample input array' + }, + ] + + assert 'inputs' in doc_data + assert doc_data['inputs'] == expected_inputs + expected_return_type = 'object' assert 'return_type' in doc_data - assert doc_data['return_type'] in expected_return_type + assert doc_data['return_type'] == expected_return_type expected_returns = [ { @@ -311,3 +365,67 @@ assert 'examples' in doc_data assert doc_data['examples'] == expected_examples + + +@api_route(r'/post/endpoint/', 'api-1-post-endpoint', + methods=['POST']) +@api_doc('/post/endpoint/', post_only=True) +def apidoc_test_post_endpoint(request): + """ + .. http:post:: /api/1/post/endpoint/ + + Endpoint documentation + + :jsonarr string type: swh object type + :>jsonarr string sha1_git: swh object sha1_git + :>jsonarr boolean found: whether the object was found or not + + """ + pass + + +def test_apidoc_input_output_doc(client): + url = reverse('api-1-post-endpoint-doc') + rv = client.get(url, HTTP_ACCEPT='text/html') + assert rv.status_code == 200, rv.content + assert_template_used(rv, 'api/apidoc.html') + + input_html_doc = ( + '
    \n' + '
    array
    \n' + '
    \n' + '

    \n' + ' \n' + ' Input array of pids\n' + ' \n' + ' \n' + '

    \n' + '
    \n' + '
    \n' + ) + + output_html_doc = ( + '
    \n' + '
    array
    \n' + '
    \n' + '

    \n' + ' \n' + ' an array of objects containing the following keys:\n' + ' \n' + ' \n' + '

      \n' + '

    • type (string): swh object type

      • \n' + '

    • sha1_git (string): swh object sha1_git

      • \n' # noqa + '

    • found (boolean): whether the object was found or not

      • \n' # noqa + '
    \n' + '
    \n' + ' \n' + '

    \n' + '
    \n' + '
    ' + ) + + assert_contains(rv, input_html_doc) + assert_contains(rv, output_html_doc)