diff --git a/debian/control b/debian/control index ff6f3ef32..722befe29 100644 --- a/debian/control +++ b/debian/control @@ -1,35 +1,38 @@ Source: swh-web-ui Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, libjs-cryptojs, libjs-jquery-datatables, libjs-jquery-flot, libjs-jquery-flot-tooltip, python3-all, python3-blinker, python3-docutils, + python3-flask-limiter, python3-flask-testing, + python3-hiredis, python3-nose, python3-pygments, + python3-redis, python3-setuptools, python3-swh.core (>= 0.0.20~), python3-swh.storage (>= 0.0.77~), python3-vcversioner, python3-yaml Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DWUI/ Package: python3-swh.web.ui Architecture: all Depends: libjs-cryptojs, libjs-jquery-datatables, libjs-jquery-flot, libjs-jquery-flot-tooltip, python3-swh.core (>= 0.0.20~), python3-swh.storage (>= 0.0.77~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage Web UI diff --git a/requirements.txt b/requirements.txt index acf85e759..548042bbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,21 +1,24 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html # Runtime dependencies Flask +Flask_Limiter swh.core >= 0.0.20 swh.storage >= 0.0.77 dateutil docutils pygments +redis +hiredis # Test dependencies #Flask-Testing #blinker # Non-Python dependencies #libjs-cryptojs #libjs-jquery-flot #libjs-jquery-flot-tooltip #libjs-jquery-datatables diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index c93692eb9..6924679ed 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,149 +1,172 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from flask import Flask +from flask_limiter import Limiter +from flask_limiter.util import get_remote_address + from swh.core import config from swh.web.ui.renderers import urlize_api_links, safe_docstring_display from swh.web.ui.renderers import revision_id_from_url, highlight_source from swh.web.ui.renderers import SWHMultiResponse, urlize_header_links from swh.web.ui.renderers import escape_author_fields from swh.storage import get_storage DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 6543), 'secret_key': ('string', 'development key'), 'max_log_revs': ('int', 25), + 'limiter': ('dict', { + 'global_limits': ['1 per minute'], + 'headers_enabled': True, + 'strategy': 'moving-window', + 'storage_uri': 'memory://', + 'storage_options': {}, + 'in_memory_fallback': ['1 per minute'], + }), } class SWHFlask(Flask): """SWH's flask application. """ response_class = SWHMultiResponse app = SWHFlask(__name__) app.add_template_filter(urlize_api_links) app.add_template_filter(urlize_header_links) app.add_template_filter(safe_docstring_display) app.add_template_filter(revision_id_from_url) app.add_template_filter(highlight_source) app.add_template_filter(escape_author_fields) def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir') conf['storage'] = get_storage(**conf['storage']) return conf - def load_controllers(): """Load the controllers for the application. """ from swh.web.ui import views, apidoc # flake8: noqa def rules(): """Returns rules from the application in dictionary form. Beware, must be called after swh.web.ui.main.load_controllers funcall. Returns: Generator of application's rules. """ for rule in app.url_map._rules: yield {'rule': rule.rule, 'methods': rule.methods, 'endpoint': rule.endpoint} def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] +def prepare_limiter(): + """Prepare Flask Limiter from configuration and App configuration""" + limiter = Limiter( + app, + key_func=get_remote_address, + **app.config['conf']['limiter'], + ) + app.limiter = limiter + def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration. Note: This function is called on a per-request basis so beware the side effects here! """ load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.yml' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf + prepare_limiter() + logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Note: This is called only once (contrast with the production mode in run_from_webserver function) Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5000 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') + prepare_limiter() + log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug) diff --git a/swh/web/ui/tests/test_app.py b/swh/web/ui/tests/test_app.py index 8efbc50ac..c1045d534 100644 --- a/swh/web/ui/tests/test_app.py +++ b/swh/web/ui/tests/test_app.py @@ -1,91 +1,102 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Functions defined here are NOT DESIGNED FOR PRODUCTION import unittest from swh.storage.api.client import RemoteStorage as Storage from swh.web.ui import main from flask_testing import TestCase # Because the Storage's __init__ function does side effect at startup... class RemoteStorageAdapter(Storage): def __init__(self, base_url): self.base_url = base_url def _init_mock_storage(base_url='https://somewhere.org:4321'): """Instanciate a remote storage whose goal is to be mocked in a test context. NOT FOR PRODUCTION Returns: An instance of swh.storage.api.client.RemoteStorage destined to be mocked (it does not do any rest call) """ return RemoteStorageAdapter(base_url) # destined to be used as mock def create_app(base_url='https://somewhere.org:4321'): """Function to initiate a flask app with storage designed to be mocked. Returns: Tuple: - app test client (for testing api, client decorator from flask) - application's full configuration - the storage instance to stub and mock - the main app without any decoration NOT FOR PRODUCTION """ storage = _init_mock_storage(base_url) # inject the mock data - conf = {'storage': storage, - 'max_log_revs': 25} + conf = { + 'storage': storage, + 'max_log_revs': 25, + 'limiter': { + 'global_limits': ['10 per minute'], + 'headers_enabled': True, + 'strategy': 'moving-window', + 'storage_uri': 'memory://', + 'storage_options': {}, + 'in_memory_fallback': ['10 per minute'], + }, + } main.app.config.update({'conf': conf}) if not main.app.config['TESTING']: # HACK: install controllers only once! main.app.config['TESTING'] = True main.load_controllers() + main.prepare_limiter() return main.app.test_client(), main.app.config, storage, main.app class SWHApiTestCase(unittest.TestCase): """Testing API class. """ @classmethod def setUpClass(cls): cls.app, cls.app_config, cls.storage, _ = create_app() cls.maxDiff = None class SWHViewTestCase(TestCase): """Testing view class. cf. http://pythonhosted.org/Flask-Testing/ """ # This inhibits template rendering # render_templates = False def create_app(self): """Initialize a Flask-Testing application instance to test view without template rendering """ _, _, _, appToDecorate = create_app() return appToDecorate class SWHApidocTestCase(SWHViewTestCase, SWHApiTestCase): """Testing APIDoc class. """