diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index 54ee9759..79c5dfe4 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,148 +1,150 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from flask.ext.api import FlaskAPI from swh.core import config from swh.web.ui.renderers import RENDERERS, urlize_api_links +from swh.web.ui.renderers import safe_docstring_display from swh.storage import get_storage DEFAULT_CONFIG = { 'storage_args': ('list[str]', ['http://localhost:5000/']), 'storage_class': ('str', 'remote_storage'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 6543), 'secret_key': ('string', 'development key'), 'max_upload_size': ('int', 16 * 1024 * 1024), 'upload_folder': ('string', '/tmp/swh-web-ui/uploads'), 'upload_allowed_extensions': ('list[str]', []) # means all are accepted } # api's definition app = FlaskAPI(__name__) app.jinja_env.filters['urlize_api_links'] = urlize_api_links +app.jinja_env.filters['safe_docstring_display'] = safe_docstring_display AUTODOC_ENDPOINT_INSTALLED = False def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir', 'upload_folder') conf['storage'] = get_storage(conf['storage_class'], conf['storage_args']) return conf def load_controllers(): """Load the controllers for the application. """ from swh.web.ui import api, errorhandler, views, apidoc # flake8: noqa # side-effects here (install autodoc endpoints so do it only once!) global AUTODOC_ENDPOINT_INSTALLED if not AUTODOC_ENDPOINT_INSTALLED: apidoc.install_browsable_api_endpoints() AUTODOC_ENDPOINT_INSTALLED = True def rules(): """Returns rules from the application in dictionary form. Beware, must be called after swh.web.ui.main.load_controllers funcall. Returns: Generator of application's rules. """ for rule in app.url_map._rules: yield {'rule': rule.rule, 'methods': rule.methods, 'endpoint': rule.endpoint} def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration. Note: This function is called on a per-request basis so beware the side effects here! """ load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.ini' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Note: This is called only once (contrast with the production mode in run_from_webserver function) Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf app.config['MAX_CONTENT_LENGTH'] = conf['max_upload_size'] app.config['DEFAULT_RENDERERS'] = RENDERERS host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug) diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py index c8fb6b10..d7576a2f 100644 --- a/swh/web/ui/renderers.py +++ b/swh/web/ui/renderers.py @@ -1,137 +1,145 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import re import yaml from flask import make_response, request from flask.ext.api import renderers, parsers from flask_api.mediatypes import MediaType from swh.web.ui import utils class SWHFilterEnricher(): """Global filter on fields. """ def filter_by_fields(self, data): """Extract a request parameter 'fields' if it exists to permit the filtering on the data dict's keys. If such field is not provided, returns the data as is. """ fields = request.args.get('fields') if fields: fields = set(fields.split(',')) data = utils.filter_field_keys(data, fields) return data class YAMLRenderer(renderers.BaseRenderer, SWHFilterEnricher): """Renderer for application/yaml. Orchestrate from python data structure to yaml. """ media_type = 'application/yaml' def render(self, data, media_type, **options): data = self.filter_by_fields(data) return yaml.dump(data, encoding=self.charset) class JSONPEnricher(): """JSONP rendering. """ def enrich_with_jsonp(self, data): """Defines a jsonp function that extracts a potential 'callback' request parameter holding the function name and wraps the data inside a call to such function e.g: GET /blah/foo/bar renders: {'output': 'wrapped'} GET /blah/foo/bar?callback=fn renders: fn({'output': 'wrapped'}) """ jsonp = request.args.get('callback') if jsonp: return '%s(%s)' % (jsonp, data) return data class SWHJSONRenderer(renderers.JSONRenderer, SWHFilterEnricher, JSONPEnricher): """Renderer for application/json. Serializes in json the data and returns it. Also deals with jsonp. If callback is found in request parameter, wrap the result as a function with name the value of the parameter query 'callback'. """ def render(self, data, media_type, **options): data = self.filter_by_fields(data) res = super().render(data, media_type, **options) return self.enrich_with_jsonp(res) def urlize_api_links(content): """Utility function for decorating api links in browsable api.""" return re.sub(r'"(/api/.*)"', r'"\1"', content) +def safe_docstring_display(docstring): + """Utility function to safely decorate docstring in browsable api.""" + print(docstring) + src = r'(Args|Raises|Throws|Returns):?.*' + dest = r'\1:
  ' + return re.sub(src, dest, docstring) + + class SWHBrowsableAPIRenderer(renderers.BrowsableAPIRenderer): """SWH's browsable api renderer. """ template = "api.html" RENDERERS = [ 'swh.web.ui.renderers.SWHJSONRenderer', 'swh.web.ui.renderers.SWHBrowsableAPIRenderer', 'flask.ext.api.parsers.URLEncodedParser', 'swh.web.ui.renderers.YAMLRenderer', ] RENDERERS_INSTANCE = [ SWHJSONRenderer(), SWHBrowsableAPIRenderer(), parsers.URLEncodedParser(), YAMLRenderer(), ] RENDERERS_BY_TYPE = { r.media_type: r for r in RENDERERS_INSTANCE } def error_response(default_error_msg, error_code, error): """Private function to create a custom error response. """ # if nothing is requested by client, use json default_application_type = 'application/json' accept_type = request.headers.get('Accept', default_application_type) renderer = RENDERERS_BY_TYPE.get( accept_type, RENDERERS_BY_TYPE[default_application_type]) # for edge cases, use the elected renderer's media type accept_type = renderer.media_type response = make_response(default_error_msg, error_code) response.headers['Content-Type'] = accept_type response.data = renderer.render({"error": str(error)}, media_type=MediaType(accept_type), status=error_code, headers={'Content-Type': accept_type}) return response diff --git a/swh/web/ui/templates/api.html b/swh/web/ui/templates/api.html index f255d8c4..629e6a74 100644 --- a/swh/web/ui/templates/api.html +++ b/swh/web/ui/templates/api.html @@ -1,190 +1,194 @@ {% block head %} {% block meta %} {% endblock %} {% block title %}Software Heritage API{% endblock %} {% block style %} {% block bootstrap_theme %} {% endblock %} {% endblock %} {% endblock %}
{% block navbar %}
- {% block branding %}Software Heritage API v1{% endblock %} + {% block branding %}Software Heritage API v1{% endblock %}
    {% block userlinks %}
    • {% endblock %}
{% endblock %}
{% if 'GET' in allowed_methods %}
GET
    {% for format in available_formats %}
  • {{ format }}
    • {% endfor %}
{% endif %} {% if 'DELETE' in allowed_methods %}
{% endif %}

{{ view_name }}

- {% if view_description %}
{{ view_description|safe }}
{% endif %} + {% if view_description %} +
+ {{ view_description | safe_docstring_display | safe}} +
+ {% endif %}
- 
{{ request.method }} {{ request.full_path }}
+ 
{{ request.method }} {{ request.full_path }}
- 
HTTP {{ status }}{% autoescape off %}
+                
HTTP {{ status }}{% autoescape off %}
 {% for key, val in headers.items() %}{{ key }}: {{ val|e }}
 {% endfor %}
 
{% if content %}{{ content|urlize_api_links }}{% endif %}
{% endautoescape %}
{% if 'POST' in allowed_methods or 'PUT' in allowed_methods or 'PATCH' in allowed_methods %}
{% if 'POST' in allowed_methods %} {% endif %} {% if 'PUT' in allowed_methods %} {% endif %} {% if 'PATCH' in allowed_methods %} {% endif %}
{% endif %}
{% block footer %} {% endblock %} {% block script %} {% endblock %} diff --git a/swh/web/ui/tests/test_renderers.py b/swh/web/ui/tests/test_renderers.py index 015d9470..8cce1a92 100644 --- a/swh/web/ui/tests/test_renderers.py +++ b/swh/web/ui/tests/test_renderers.py @@ -1,192 +1,219 @@ # Copyright (C) 2015 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import json import unittest import yaml from flask_api.mediatypes import MediaType from nose.tools import istest from unittest.mock import patch from swh.web.ui import renderers class RendererTestCase(unittest.TestCase): @patch('swh.web.ui.renderers.request') @istest def SWHFilterRenderer_do_nothing(self, mock_request): # given mock_request.args = {} swhFilterRenderer = renderers.SWHFilterEnricher() input_data = {'a': 'some-data'} # when actual_data = swhFilterRenderer.filter_by_fields(input_data) # then self.assertEquals(actual_data, input_data) @patch('swh.web.ui.renderers.utils') @patch('swh.web.ui.renderers.request') @istest def SWHFilterRenderer_do_filter(self, mock_request, mock_utils): # given mock_request.args = {'fields': 'a,c'} mock_utils.filter_field_keys.return_value = {'a': 'some-data'} swhFilterRenderer = renderers.SWHFilterEnricher() input_data = {'a': 'some-data', 'b': 'some-other-data'} # when actual_data = swhFilterRenderer.filter_by_fields(input_data) # then self.assertEquals(actual_data, {'a': 'some-data'}) mock_utils.filter_field_keys.assert_called_once_with(input_data, {'a', 'c'}) @patch('swh.web.ui.renderers.request') @istest def yamlRenderer_without_filter(self, mock_request): # given mock_request.args = {} yamlRenderer = renderers.YAMLRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = input_data # when actual_data = yamlRenderer.render(input_data, 'application/yaml') # then self.assertEqual(yaml.load(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def yamlRenderer(self, mock_request): # given mock_request.args = {'fields': 'type,target'} yamlRenderer = renderers.YAMLRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = {'target': 'sha1-dir', 'type': 'dir'} # when actual_data = yamlRenderer.render(input_data, 'application/yaml') # then self.assertEqual(yaml.load(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def jsonRenderer_basic(self, mock_request): # given mock_request.args = {} jsonRenderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = input_data # when actual_data = jsonRenderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(json.loads(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def jsonRenderer_basic_with_filter(self, mock_request): # given mock_request.args = {'fields': 'target'} jsonRenderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} expected_data = {'target': 'sha1-dir'} # when actual_data = jsonRenderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(json.loads(actual_data), expected_data) @patch('swh.web.ui.renderers.request') @istest def jsonRenderer_basic_with_filter_and_jsonp(self, mock_request): # given mock_request.args = {'fields': 'target', 'callback': 'jsonpfn'} jsonRenderer = renderers.SWHJSONRenderer() input_data = {'target': 'sha1-dir', 'type': 'dir', 'dir-id': 'dir-id-sha1-git'} # when actual_data = jsonRenderer.render(input_data, MediaType( 'application/json')) # then self.assertEqual(actual_data, 'jsonpfn({"target": "sha1-dir"})') @patch('swh.web.ui.renderers.request') @istest def jsonpEnricher_basic_with_filter_and_jsonp(self, mock_request): # given mock_request.args = {'callback': 'jsonpfn'} jsonpEnricher = renderers.JSONPEnricher() # when actual_output = jsonpEnricher.enrich_with_jsonp({'output': 'test'}) # then self.assertEqual(actual_output, "jsonpfn({'output': 'test'})") @patch('swh.web.ui.renderers.request') @istest def jsonpEnricher_do_nothing(self, mock_request): # given mock_request.args = {} jsonpEnricher = renderers.JSONPEnricher() # when actual_output = jsonpEnricher.enrich_with_jsonp({'output': 'test'}) # then self.assertEqual(actual_output, {'output': 'test'}) @istest def urlize_api_links(self): # update api link with html links content with links content = '{"url": "/api/1/abc/"}' expected_content = '{"url": "/api/1/abc/"}' self.assertEquals(renderers.urlize_api_links(content), expected_content) # will do nothing since it's not an api url other_content = '{"url": "/something/api/1/other"}' self.assertEquals(renderers.urlize_api_links(other_content), other_content) + + @istest + def safe_docstring_display(self): + # update api link with html links content with links + docstring = """

Show all revisions (~git log) starting from +sha1_git. + The first element returned is the given sha1_git.

+

Args: + sha1_git: the revision's hash

+

Returns: + Information on the revision if found.

+

Raises: + BadInputExc in case of unknown algo_hash or bad hash + NotFoundExc if the revision is not found.

""" + expected_docstring = """

Show all revisions (~git log) starting from +sha1_git. + The first element returned is the given sha1_git.

+

Args:
   + sha1_git: the revision's hash

+

Returns:
   + Information on the revision if found.

+

Raises:
   + BadInputExc in case of unknown algo_hash or bad hash + NotFoundExc if the revision is not found.

""" + + self.assertEquals(renderers.safe_docstring_display(docstring), + expected_docstring)