diff --git a/swh/web/ui/apidoc.py b/swh/web/ui/apidoc.py
index 2c15838d5..d2cf08072 100644
--- a/swh/web/ui/apidoc.py
+++ b/swh/web/ui/apidoc.py
@@ -1,276 +1,238 @@
# Copyright (C) 2015 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
-import yaml
-import json
from functools import wraps
from flask import request, render_template, url_for
+from flask import g
from swh.web.ui.main import app
class argtypes(object):
"""Class for centralizing argument type descriptions
"""
ts = 'timestamp'
int = 'integer'
path = 'path'
sha1 = 'sha1'
uuid = 'uuid'
sha1_git = 'sha1_git'
octet_stream = 'octet stream'
algo_and_hash = 'algo_hash:hash'
class rettypes(object):
"""Class for centralizing return type descriptions
"""
list = 'list'
dict = 'dict'
class excs(object):
"""Class for centralizing exception type descriptions
"""
badinput = 'BadInputExc'
notfound = 'NotFoundExc'
class APIUrls(object):
"""
Class to manage API documentation URLs.
* Indexes all routes documented using apidoc's decorators.
* Tracks endpoint/request processing method relationships for use
in generating related urls in API documentation
Relies on the load_controllers logic in main.py for initialization.
"""
apidoc_routes = {}
method_endpoints = {}
@classmethod
def get_app_endpoints(cls):
return cls.apidoc_routes
@classmethod
def get_method_endpoints(cls, fname):
if len(cls.method_endpoints) == 0:
cls.method_endpoints = cls.group_routes_by_method()
return cls.method_endpoints[fname]
@classmethod
def group_routes_by_method(cls):
"""
Group URL endpoints according to their processing method.
Returns:
A dict where keys are the processing method names, and values
are the routes that are bound to the key method.
"""
endpoints = {}
for rule in app.url_map.iter_rules():
rule_dict = {'rule': rule.rule,
'methods': rule.methods}
if rule.endpoint not in endpoints:
endpoints[rule.endpoint] = [rule_dict]
else:
endpoints[rule.endpoint].append(rule_dict)
return endpoints
@classmethod
def index_add_route(cls, route, docstring):
"""
Add a route to the self-documenting API reference
"""
if route not in cls.apidoc_routes:
cls.apidoc_routes[route] = docstring
class route(object):
"""
Decorate an API method to register it in the API doc route index
and create the corresponding Flask route.
Caution: decorating a method with this requires to also decorate it
__at least__ with @returns, or breaks the decorated endpoint
Args:
route: the documentation page's route
noargs: set to True if the route has no arguments, and its result
should be displayed anytime its documentation is requested
"""
def __init__(self, route, noargs=False):
self.route = route
self.noargs = noargs
def __call__(self, f):
APIUrls.index_add_route(self.route, f.__doc__)
@wraps(f)
def doc_func(*args, **kwargs):
return f(call_args=(args, kwargs),
doc_route=self.route,
noargs=self.noargs)
if not self.noargs:
app.add_url_rule(self.route, f.__name__, doc_func)
return doc_func
class arg(object):
"""
Decorate an API method to display an argument's information on the doc
page specified by @route above.
Args:
name: the argument's name. MUST match the method argument's name to
create the example request URL.
default: the argument's default value
argtype: the argument's type (map, dict, list, tuple...)
argdoc: the argument's documentation string
"""
def __init__(self, name, default, argtype, argdoc):
self.doc_dict = {
'name': name,
'type': argtype,
'doc': argdoc,
'default': default
}
def __call__(self, f):
@wraps(f)
def arg_fun(*args, **kwargs):
if 'args' in kwargs:
kwargs['args'].append(self.doc_dict)
else:
kwargs['args'] = [self.doc_dict]
return f(*args, **kwargs)
return arg_fun
class raises(object):
"""
Decorate an API method to display information pertaining to an exception
that can be raised by this method.
Args:
exc: the exception name
doc: the exception's documentation string
"""
def __init__(self, exc, doc):
self.exc_dict = {
'exc': exc,
'doc': doc
}
def __call__(self, f):
@wraps(f)
def exc_fun(*args, **kwargs):
if 'excs' in kwargs:
kwargs['excs'].append(self.exc_dict)
else:
kwargs['excs'] = [self.exc_dict]
return f(*args, **kwargs)
return exc_fun
-def make_response_from_mimetype(rv, env):
-
- def wants_html(best_match):
- return best_match == 'text/html' and \
- request.accept_mimetypes[best_match] > \
- request.accept_mimetypes['application/json']
-
- def wants_yaml(best_match):
- return best_match == 'application/yaml' and \
- request.accept_mimetypes[best_match] > \
- request.accept_mimetypes['application/json']
-
- if isinstance(rv, dict) or isinstance(rv, list):
- acc_mime = ['application/json', 'application/yaml', 'text/html']
- best_match = request.accept_mimetypes.best_match(acc_mime)
- # return a template render
- if wants_html(best_match):
- data = json.dumps(rv, sort_keys=True,
- indent=4, separators=(',', ': '))
- env['response_data'] = data
- env['request'] = request
- rv = app.response_class(render_template('apidoc.html', **env),
- content_type='text/html')
- # return formatted yaml
- elif wants_yaml(best_match):
- rv = app.response_class(
- yaml.dump(rv),
- content_type='application/yaml')
- # return formatted json
- else:
- # jsonify is unhappy with lists in Flask 0.10.1, use json.dumps
- rv = app.response_class(
- json.dumps(rv),
- content_type='application/json')
- return rv
-
-
class returns(object):
"""
Decorate an API method to display information about its return value.
Caution: this MUST be the last decorator in the apidoc decorator stack,
or the decorated endpoint breaks
Args:
rettype: the return value's type (map, dict, list, tuple...)
retdoc: the return value's documentation string
"""
def __init__(self, rettype=None, retdoc=None):
self.return_dict = {
'type': rettype,
'doc': retdoc
}
def filter_api_url(self, endpoint, route_re, noargs):
doc_methods = {'GET', 'HEAD', 'OPTIONS'}
if re.match(route_re, endpoint['rule']):
if endpoint['methods'] == doc_methods and not noargs:
return False
return True
def __call__(self, f):
@wraps(f)
def ret_fun(*args, **kwargs):
# Build documentation
env = {
'docstring': f.__doc__,
'route': kwargs['doc_route'],
'return': self.return_dict
}
for arg in ['args', 'excs']:
if arg in kwargs:
env[arg] = kwargs[arg]
route_re = re.compile('.*%s$' % kwargs['doc_route'])
endpoint_list = APIUrls.get_method_endpoints(f.__name__)
other_urls = [url for url in endpoint_list if
self.filter_api_url(url, route_re, kwargs['noargs'])]
env['urls'] = other_urls
# Build example endpoint URL
if 'args' in env:
defaults = {arg['name']: arg['default'] for arg in env['args']}
env['example'] = url_for(f.__name__, **defaults)
# Prepare and send to mimetype selector if it's not a doc request
if re.match(route_re, request.url) and not kwargs['noargs']:
return app.response_class(
render_template('apidoc.html', **env),
content_type='text/html')
cargs, ckwargs = kwargs['call_args']
- rv = f(*cargs, **ckwargs)
- return make_response_from_mimetype(rv, env)
+ g.doc_env = env # Store for response processing
+ return f(*cargs, **ckwargs)
return ret_fun
diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py
index 00c5bac45..287774242 100644
--- a/swh/web/ui/main.py
+++ b/swh/web/ui/main.py
@@ -1,139 +1,141 @@
# Copyright (C) 2015 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import logging
import os
import json
from flask import Flask
from swh.core import config
from swh.web.ui.renderers import RENDERERS, urlize_api_links
from swh.web.ui.renderers import safe_docstring_display
from swh.web.ui.renderers import revision_id_from_url
+from swh.web.ui.renderers import SWHMultiResponse
from swh.storage import get_storage
DEFAULT_CONFIG = {
'storage_args': ('list[str]', ['http://localhost:5000/']),
'storage_class': ('str', 'remote_storage'),
'log_dir': ('string', '/tmp/swh/log'),
'debug': ('bool', None),
'host': ('string', '127.0.0.1'),
'port': ('int', 6543),
'secret_key': ('string', 'development key'),
'max_log_revs': ('int', 25),
}
# api's definition
app = Flask(__name__)
+app.response_class = SWHMultiResponse
app.jinja_env.filters['urlize_api_links'] = urlize_api_links
app.jinja_env.filters['safe_docstring_display'] = safe_docstring_display
app.jinja_env.filters['revision_id_from_url'] = revision_id_from_url
def read_config(config_file):
"""Read the configuration file `config_file`, update the app with
parameters (secret_key, conf) and return the parsed configuration as a
dict"""
conf = config.read(config_file, DEFAULT_CONFIG)
config.prepare_folders(conf, 'log_dir')
conf['storage'] = get_storage(conf['storage_class'], conf['storage_args'])
return conf
def load_controllers():
"""Load the controllers for the application.
"""
from swh.web.ui import views, apidoc # flake8: noqa
def rules():
"""Returns rules from the application in dictionary form.
Beware, must be called after swh.web.ui.main.load_controllers funcall.
Returns:
Generator of application's rules.
"""
for rule in app.url_map._rules:
yield {'rule': rule.rule,
'methods': rule.methods,
'endpoint': rule.endpoint}
def storage():
"""Return the current application's storage.
"""
return app.config['conf']['storage']
def run_from_webserver(environ, start_response):
"""Run the WSGI app from the webserver, loading the configuration.
Note: This function is called on a per-request basis so beware the side
effects here!
"""
load_controllers()
config_path = '/etc/softwareheritage/webapp/webapp.ini'
conf = read_config(config_path)
app.secret_key = conf['secret_key']
app.config['conf'] = conf
app.config['DEFAULT_RENDERERS'] = RENDERERS
logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'),
level=logging.INFO)
return app(environ, start_response)
def run_debug_from(config_path, verbose=False):
"""Run the api's server in dev mode.
Note: This is called only once (contrast with the production mode
in run_from_webserver function)
Args:
conf is a dictionary of keywords:
- 'db_url' the db url's access (through psycopg2 format)
- 'content_storage_dir' revisions/directories/contents storage on disk
- 'host' to override the default 127.0.0.1 to open or not the server
to the world
- 'port' to override the default of 5000 (from the underlying layer:
flask)
- 'debug' activate the verbose logs
- 'secret_key' the flask secret key
Returns:
Never
"""
load_controllers()
conf = read_config(config_path)
app.secret_key = conf['secret_key']
app.config['conf'] = conf
app.config['DEFAULT_RENDERERS'] = RENDERERS
host = conf.get('host', '127.0.0.1')
port = conf.get('port')
debug = conf.get('debug')
log_file = os.path.join(conf['log_dir'], 'web-ui.log')
logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO,
handlers=[logging.FileHandler(log_file),
logging.StreamHandler()])
app.run(host=host, port=port, debug=debug)
diff --git a/swh/web/ui/renderers.py b/swh/web/ui/renderers.py
index 3b128d2da..140a1f0e2 100644
--- a/swh/web/ui/renderers.py
+++ b/swh/web/ui/renderers.py
@@ -1,150 +1,207 @@
# Copyright (C) 2015 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import re
import yaml
+import json
-from flask import make_response, request
+from flask import make_response, request, Response, render_template
+from flask import g
from flask.ext.api import renderers, parsers
from flask_api.mediatypes import MediaType
from swh.web.ui import utils
class SWHFilterEnricher():
"""Global filter on fields.
"""
def filter_by_fields(self, data):
"""Extract a request parameter 'fields' if it exists to permit the
filtering on the data dict's keys.
If such field is not provided, returns the data as is.
"""
fields = request.args.get('fields')
if fields:
fields = set(fields.split(','))
data = utils.filter_field_keys(data, fields)
return data
class YAMLRenderer(renderers.BaseRenderer, SWHFilterEnricher):
"""Renderer for application/yaml.
Orchestrate from python data structure to yaml.
"""
media_type = 'application/yaml'
def render(self, data, media_type, **options):
data = self.filter_by_fields(data)
return yaml.dump(data, encoding=self.charset)
class JSONPEnricher():
"""JSONP rendering.
"""
def enrich_with_jsonp(self, data):
"""Defines a jsonp function that extracts a potential 'callback'
request parameter holding the function name and wraps the data
inside a call to such function
e.g:
GET /blah/foo/bar renders: {'output': 'wrapped'}
GET /blah/foo/bar?callback=fn renders: fn({'output': 'wrapped'})
"""
jsonp = request.args.get('callback')
if jsonp:
return '%s(%s)' % (jsonp, data)
return data
class SWHJSONRenderer(renderers.JSONRenderer,
SWHFilterEnricher,
JSONPEnricher):
"""Renderer for application/json.
Serializes in json the data and returns it.
Also deals with jsonp. If callback is found in request parameter,
wrap the result as a function with name the value of the parameter
query 'callback'.
"""
def render(self, data, media_type, **options):
data = self.filter_by_fields(data)
res = super().render(data, media_type, **options)
return self.enrich_with_jsonp(res)
+class SWHMultiResponse(Response, SWHFilterEnricher):
+ """
+ A Flask Response subclass.
+ Override force_type to transform dict responses into callable Flask
+ response objects whose mimetype matches the request's Accept header: HTML
+ template render, YAML dump or default to a JSON dump.
+ """
+
+ @classmethod
+ def make_response_from_mimetype(cls, rv):
+
+ def wants_html(best_match):
+ return best_match == 'text/html' and \
+ request.accept_mimetypes[best_match] > \
+ request.accept_mimetypes['application/json']
+
+ def wants_yaml(best_match):
+ return best_match == 'application/yaml' and \
+ request.accept_mimetypes[best_match] > \
+ request.accept_mimetypes['application/json']
+
+ if isinstance(rv, dict) or isinstance(rv, list):
+ rv = cls.filter_by_fields(cls, rv)
+ acc_mime = ['application/json', 'application/yaml', 'text/html']
+ best_match = request.accept_mimetypes.best_match(acc_mime)
+ # return a template render
+ if wants_html(best_match):
+ data = json.dumps(rv, sort_keys=True,
+ indent=4, separators=(',', ': '))
+ env = g.get('doc_env', {})
+ env['response_data'] = data
+ env['request'] = request
+ rv = Response(render_template('apidoc.html', **env),
+ content_type='text/html')
+ # return formatted yaml
+ elif wants_yaml(best_match):
+ rv = Response(
+ yaml.dump(rv),
+ content_type='application/yaml')
+ # return formatted json
+ else:
+ # jsonify is unhappy with lists in Flask 0.10.1, use json.dumps
+ rv = Response(
+ json.dumps(rv),
+ content_type='application/json')
+ return rv
+
+ @classmethod
+ def force_type(cls, rv, environ=None):
+ # Data from apidoc
+ if isinstance(rv, dict) or isinstance(rv, list):
+ rv = cls.make_response_from_mimetype(rv)
+ return super().force_type(rv, environ)
+
+
def urlize_api_links(content):
"""Utility function for decorating api links in browsable api."""
return re.sub(r'"(/api/.*|/browse/.*)"', r'"\1"', content)
def safe_docstring_display(docstring):
"""Utility function to safely decorate docstring in browsable api."""
src = r'(Args|Raises?|Throws?|Yields?|Returns?|Examples?|Samples?):.*'
dest = r'
\1:
'
return re.sub(src, dest, docstring)
def revision_id_from_url(url):
"""Utility function to obtain a revision's ID from its browsing URL."""
return re.sub(r'/browse/revision/([0-9a-f]{40}|[0-9a-f]{64})/.*',
r'\1', url)
class SWHBrowsableAPIRenderer(renderers.BrowsableAPIRenderer):
"""SWH's browsable api renderer.
"""
template = "api.html"
RENDERERS = [
'swh.web.ui.renderers.SWHJSONRenderer',
'swh.web.ui.renderers.SWHBrowsableAPIRenderer',
'flask.ext.api.parsers.URLEncodedParser',
'swh.web.ui.renderers.YAMLRenderer',
]
RENDERERS_INSTANCE = [
SWHJSONRenderer(),
SWHBrowsableAPIRenderer(),
parsers.URLEncodedParser(),
YAMLRenderer(),
]
RENDERERS_BY_TYPE = {
r.media_type: r
for r in RENDERERS_INSTANCE
}
def error_response(default_error_msg, error_code, error):
"""Private function to create a custom error response.
"""
# if nothing is requested by client, use json
default_application_type = 'application/json'
accept_type = request.headers.get('Accept', default_application_type)
renderer = RENDERERS_BY_TYPE.get(
accept_type,
RENDERERS_BY_TYPE[default_application_type])
# for edge cases, use the elected renderer's media type
accept_type = renderer.media_type
response = make_response(default_error_msg, error_code)
response.headers['Content-Type'] = accept_type
response.data = renderer.render({"error": str(error)},
media_type=MediaType(accept_type),
status=error_code,
headers={'Content-Type': accept_type})
return response
diff --git a/swh/web/ui/templates/origin.html b/swh/web/ui/templates/origin.html
index 963b0377e..3d773559d 100644
--- a/swh/web/ui/templates/origin.html
+++ b/swh/web/ui/templates/origin.html
@@ -1,40 +1,40 @@
{% extends "layout.html" %}
{% block title %}Origin{% endblock %}
{% block content %}
{% if message is not none %}
{{ message }}
{% endif %}
{% if origin is not none %}
Details on origin {{ origin_id }}:
- {% for key in ['type', 'lister', 'projet', 'url'] %}
+ {% for key in ['type', 'lister', 'project', 'url'] %}
{% if origin[key] is not none %}
{{ key }}
{{ origin[key] }}
{% endif %}
{% endfor %}
{% if 'decoding_failures' in content %}