diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py index 7da2cb426..be575c4a4 100644 --- a/swh/web/ui/renderers.py +++ b/swh/web/ui/renderers.py @@ -1,284 +1,282 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re import yaml import json from docutils.core import publish_parts from docutils.writers.html4css1 import Writer, HTMLTranslator from inspect import cleandoc from jinja2 import Markup from flask import request, Response, render_template from flask import g from pygments import highlight from pygments.lexers import guess_lexer from pygments.formatters import HtmlFormatter from swh.web.ui import utils class SWHFilterEnricher(): """Global filter on fields. """ @classmethod def filter_by_fields(cls, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.args.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data class SWHComputeLinkHeader: """Add link header to response. Mixin intended to be used for example in SWHMultiResponse """ @classmethod def compute_link_header(cls, rv, options): """Add Link header in returned value results. Expects rv to be a dict with 'results' and 'headers' key: 'results': the returned value expected to be shown 'headers': dictionary with link-next and link-prev Args: rv (dict): with keys: - 'headers': potential headers with 'link-next' and 'link-prev' keys - 'results': containing the result to return options (dict): the initial dict to update with result if any Returns: - Dict 'options' updated with headers 'Link' containing - the 'link-next' and 'link-prev' headers. - - Otherwise, options is returned unchanged + Dict with optional keys 'link-next' and 'link-prev'. """ link_headers = [] if 'headers' not in rv: - return options + return {} rv_headers = rv['headers'] if 'link-next' in rv_headers: link_headers.append('<%s>; rel="next"' % ( rv_headers['link-next'])) if 'link-prev' in rv_headers: link_headers.append('<%s>; rel="previous"' % ( rv_headers['link-prev'])) if link_headers: link_header_str = ','.join(link_headers) headers = options.get('headers', {}) headers.update({ 'Link': link_header_str }) - options['headers'] = headers - return options + return headers - return options + return {} class SWHTransformProcessor: """Transform an eventual returned value with multiple layer of information with only what's necessary. If the returned value rv contains the 'results' key, this is the associated value which is returned. Otherwise, return the initial dict without the potential 'headers' key. """ @classmethod def transform(cls, rv): if 'results' in rv: return rv['results'] if 'headers' in rv: rv.pop('headers') return rv class SWHMultiResponse(Response, SWHFilterEnricher, SWHComputeLinkHeader, SWHTransformProcessor): """ A Flask Response subclass. Override force_type to transform dict/list responses into callable Flask response objects whose mimetype matches the request's Accept header: HTML template render, YAML dump or default to a JSON dump. """ @classmethod def make_response_from_mimetype(cls, rv, options={}): + options = options.copy() if not (isinstance(rv, list) or isinstance(rv, dict)): return rv def wants_html(best_match): return best_match == 'text/html' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] def wants_yaml(best_match): return best_match == 'application/yaml' and \ request.accept_mimetypes[best_match] > \ request.accept_mimetypes['application/json'] rv = cls.filter_by_fields(rv) acc_mime = ['application/json', 'application/yaml', 'text/html'] best_match = request.accept_mimetypes.best_match(acc_mime) - options = cls.compute_link_header(rv, options) + options['headers'] = cls.compute_link_header(rv, options) + rv = cls.transform(rv) if wants_html(best_match): data = json.dumps(rv, sort_keys=True, indent=4, separators=(',', ': ')) env = g.get('doc_env', {}) env['response_data'] = data env['headers_data'] = None if options and 'headers' in options: env['headers_data'] = options['headers'] env['request'] = request rv = Response(render_template('apidoc.html', **env), content_type='text/html', **options) elif wants_yaml(best_match): rv = Response( yaml.dump(rv), content_type='application/yaml', **options) else: # jsonify is unhappy with lists in Flask 0.10.1, use json.dumps rv = Response( json.dumps(rv), content_type='application/json', **options) return rv @classmethod def force_type(cls, rv, environ=None): if isinstance(rv, dict) or isinstance(rv, list): rv = cls.make_response_from_mimetype(rv) return super().force_type(rv, environ) def error_response(error_code, error): """Private function to create a custom error response. """ error_opts = {'status': error_code} error_data = {'error': str(error)} return SWHMultiResponse.make_response_from_mimetype(error_data, options=error_opts) def urlize_api_links(text): """Utility function for decorating api links in browsable api. Args: text: whose content matching links should be transformed into contextual API or Browse html links. Returns The text transformed if any link is found. The text as is otherwise. """ return re.sub(r'"(/api/.*|/browse/.*)"', r'"\1"', text) def urlize_header_links(text): """Utility function for decorating headers links in browsable api. Args text: Text whose content contains Link header value Returns: The text transformed with html link if any link is found. The text as is otherwise. """ return re.sub(r'<(/api/.*|/browse/.*)>', r'<\1>', text) class NoHeaderHTMLTranslator(HTMLTranslator): """ Docutils translator subclass to customize the generation of HTML from reST-formatted docstrings """ def __init__(self, document): super().__init__(document) self.body_prefix = [] self.body_suffix = [] def visit_bullet_list(self, node): self.context.append((self.compact_simple, self.compact_p)) self.compact_p = None self.compact_simple = self.is_compactable(node) self.body.append(self.starttag(node, 'ul', CLASS='docstring')) DOCSTRING_WRITER = Writer() DOCSTRING_WRITER.translator_class = NoHeaderHTMLTranslator def safe_docstring_display(docstring): """ Utility function to htmlize reST-formatted documentation in browsable api. """ docstring = cleandoc(docstring) return publish_parts(docstring, writer=DOCSTRING_WRITER)['html_body'] def revision_id_from_url(url): """Utility function to obtain a revision's ID from its browsing URL.""" return re.sub(r'/browse/revision/([0-9a-f]{40}|[0-9a-f]{64})/.*', r'\1', url) def highlight_source(source_code_as_text): """Leverage pygments to guess and highlight source code. Args source_code_as_text (str): source code in plain text Returns: Highlighted text if possible or plain text otherwise """ try: maybe_lexer = guess_lexer(source_code_as_text) if maybe_lexer: r = highlight( source_code_as_text, maybe_lexer, HtmlFormatter(linenos=True, lineanchors='l', anchorlinenos=True)) else: r = '
%s
' % source_code_as_text except: r = '
%s
' % source_code_as_text return Markup(r) diff --git a/swh/web/ui/tests/test_renderers.py b/swh/web/ui/tests/test_renderers.py index 08691c3f0..6eb43d83f 100644 --- a/swh/web/ui/tests/test_renderers.py +++ b/swh/web/ui/tests/test_renderers.py @@ -1,326 +1,324 @@ # Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import unittest import yaml from flask import Response from nose.tools import istest from unittest.mock import patch, MagicMock from swh.web.ui import renderers class SWHComputeLinkHeaderTest(unittest.TestCase): @istest def compute_link_header(self): rv = { 'headers': {'link-next': 'foo', 'link-prev': 'bar'}, 'results': [1, 2, 3] } options = {} # when - _options = renderers.SWHComputeLinkHeader.compute_link_header( + headers = renderers.SWHComputeLinkHeader.compute_link_header( rv, options) - self.assertEquals(_options, {'headers': { + self.assertEquals(headers, { 'Link': '; rel="next",; rel="previous"', - }}) + }) @istest def compute_link_header_nothing_changed(self): rv = {} options = {} # when - _options = renderers.SWHComputeLinkHeader.compute_link_header( + headers = renderers.SWHComputeLinkHeader.compute_link_header( rv, options) - self.assertEquals(_options, {}) + self.assertEquals(headers, {}) @istest def compute_link_header_nothing_changed_2(self): rv = {'headers': {}} options = {} # when - _options = renderers.SWHComputeLinkHeader.compute_link_header( + headers = renderers.SWHComputeLinkHeader.compute_link_header( rv, options) - self.assertEquals(_options, {}) + self.assertEquals(headers, {}) class SWHTransformProcessorTest(unittest.TestCase): @istest def transform_only_return_results_1(self): rv = {'results': {'some-key': 'some-value'}} self.assertEquals(renderers.SWHTransformProcessor.transform(rv), {'some-key': 'some-value'}) @istest def transform_only_return_results_2(self): rv = {'headers': {'something': 'do changes'}, 'results': {'some-key': 'some-value'}} self.assertEquals(renderers.SWHTransformProcessor.transform(rv), {'some-key': 'some-value'}) @istest def transform_do_remove_headers(self): rv = {'headers': {'something': 'do changes'}, 'some-key': 'some-value'} self.assertEquals(renderers.SWHTransformProcessor.transform(rv), {'some-key': 'some-value'}) @istest def transform_do_nothing(self): rv = {'some-key': 'some-value'} self.assertEquals(renderers.SWHTransformProcessor.transform(rv), {'some-key': 'some-value'}) class RendererTestCase(unittest.TestCase): @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.json') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.render_template') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_html(self, mock_filter, mock_render, mock_request, mock_json, mock_g): # given data = { 'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc' } mock_g.get.return_value = {'my_key': 'my_display_value'} # mock_enricher.return_value = (data, {}) mock_filter.return_value = data expected_env = { 'my_key': 'my_display_value', 'response_data': json.dumps(data), 'request': mock_request, - 'headers_data': { - 'Link': '; rel="next"' # noqa - } + 'headers_data': {}, } def mock_mimetypes(key): mimetypes = { 'text/html': 10, 'application/json': 0.1, 'application/yaml': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock(return_value='text/html') mock_request.accept_mimetypes = accept_mimetypes mock_json.dumps.return_value = json.dumps(data) # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then # mock_enricher.assert_called_once_with(data, {}) mock_filter.assert_called_once_with(data) mock_render.assert_called_with('apidoc.html', **expected_env) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'text/html') @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.yaml') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_yaml(self, mock_filter, mock_request, mock_yaml, mock_g): # given data = {'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'} def mock_mimetypes(key): mimetypes = { 'application/yaml': 10, 'application/json': 0.1, 'text/html': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock( return_value='application/yaml') mock_request.accept_mimetypes = accept_mimetypes mock_yaml.dump.return_value = yaml.dump(data) mock_filter.return_value = data # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then mock_filter.assert_called_once_with(data) mock_yaml.dump.assert_called_once_with(data) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'application/yaml') self.assertEqual(data, yaml.load(rv.data.decode('utf-8'))) @patch('swh.web.ui.renderers.g') @patch('swh.web.ui.renderers.json') @patch('swh.web.ui.renderers.request') @patch('swh.web.ui.renderers.SWHMultiResponse.filter_by_fields') @istest def swh_multi_response_mimetype_json(self, mock_filter, mock_request, mock_json, mock_g): # given data = {'data': [12, 34], 'id': 'adc83b19e793491b1c6ea0fd8b46cd9f32e592fc'} def mock_mimetypes(key): mimetypes = { 'application/json': 10, 'text/html': 0.1, 'application/yaml': 0.1 } return mimetypes[key] accept_mimetypes = MagicMock() accept_mimetypes.__getitem__.side_effect = mock_mimetypes accept_mimetypes.best_match = MagicMock( return_value='application/json') mock_request.accept_mimetypes = accept_mimetypes mock_json.dumps.return_value = json.dumps(data) mock_filter.return_value = data # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(data) # then mock_filter.assert_called_once_with(data) mock_json.dumps.assert_called_once_with(data) self.assertEqual(rv.status_code, 200) self.assertEqual(rv.mimetype, 'application/json') self.assertEqual(data, json.loads(rv.data.decode('utf-8'))) @patch('swh.web.ui.renderers.request') @istest def swh_multi_response_make_response_not_list_dict(self, mock_request): # given incoming = Response() # when rv = renderers.SWHMultiResponse.make_response_from_mimetype(incoming) # then self.assertEqual(rv, incoming) @patch('swh.web.ui.renderers.request') @istest def swh_filter_renderer_do_nothing(self, mock_request): # given mock_request.args = {} swh_filter_renderer = renderers.SWHFilterEnricher() input_data = {'a': 'some-data'} # when actual_data = swh_filter_renderer.filter_by_fields(input_data) # then self.assertEquals(actual_data, input_data) @patch('swh.web.ui.renderers.utils') @patch('swh.web.ui.renderers.request') @istest def swh_filter_renderer_do_filter(self, mock_request, mock_utils): # given mock_request.args = {'fields': 'a,c'} mock_utils.filter_field_keys.return_value = {'a': 'some-data'} swh_filter_user = renderers.SWHMultiResponse() input_data = {'a': 'some-data', 'b': 'some-other-data'} # when actual_data = swh_filter_user.filter_by_fields(input_data) # then self.assertEquals(actual_data, {'a': 'some-data'}) mock_utils.filter_field_keys.assert_called_once_with(input_data, {'a', 'c'}) @istest def urlize_api_links(self): # update api link with html links content with links content = '{"url": "/api/1/abc/"}' expected_content = '{"url": "/api/1/abc/"}' self.assertEquals(renderers.urlize_api_links(content), expected_content) # update /browse link with html links content with links content = '{"url": "/browse/def/"}' expected_content = '{"url": "' \ '/browse/def/"}' self.assertEquals(renderers.urlize_api_links(content), expected_content) # will do nothing since it's not an api url other_content = '{"url": "/something/api/1/other"}' self.assertEquals(renderers.urlize_api_links(other_content), other_content) @istest def urlize_header_links(self): # update api link with html links content with links content = """; rel="next" ; rel="prev" """ expected_content = """</api/1/abc/>; rel="next" </api/1/def/>; rel="prev" """ self.assertEquals(renderers.urlize_header_links(content), expected_content) @istest def revision_id_from_url(self): url = ('/browse/revision/9ba4bcb645898d562498ea66a0df958ef0e7a68c/' 'prev/9ba4bcb645898d562498ea66a0df958ef0e7aaaa/') expected_id = '9ba4bcb645898d562498ea66a0df958ef0e7a68c' self.assertEqual(renderers.revision_id_from_url(url), expected_id) @istest def safe_docstring_display(self): # update api link with html links content with links docstring = """This is my list header: - Here is item 1, with a continuation line right here - Here is item 2 Here is something that is not part of the list""" expected_docstring = """

This is my list header:

Here is something that is not part of the list

""" self.assertEquals(renderers.safe_docstring_display(docstring), expected_docstring)