diff --git a/PKG-INFO b/PKG-INFO index 953826c2..7f6754f6 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.web.ui -Version: 0.0.82 +Version: 0.0.83 Summary: Software Heritage Web UI Home-page: https://forge.softwareheritage.org/diffusion/DWUI/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.web.ui.egg-info/PKG-INFO b/swh.web.ui.egg-info/PKG-INFO index 953826c2..7f6754f6 100644 --- a/swh.web.ui.egg-info/PKG-INFO +++ b/swh.web.ui.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.web.ui -Version: 0.0.82 +Version: 0.0.83 Summary: Software Heritage Web UI Home-page: https://forge.softwareheritage.org/diffusion/DWUI/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh/web/ui/main.py b/swh/web/ui/main.py index 4efdb3ae..0d01674a 100644 --- a/swh/web/ui/main.py +++ b/swh/web/ui/main.py @@ -1,184 +1,201 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information +import ipaddress import logging import os from flask import Flask from flask_limiter import Limiter from flask_limiter.util import get_remote_address from swh.core import config from swh.web.ui.renderers import urlize_api_links, safe_docstring_display from swh.web.ui.renderers import revision_id_from_url, highlight_source from swh.web.ui.renderers import SWHMultiResponse, urlize_header_links from swh.storage import get_storage DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', None), 'host': ('string', '127.0.0.1'), 'port': ('int', 5004), 'secret_key': ('string', 'development key'), 'max_log_revs': ('int', 25), 'limiter': ('dict', { 'global_limits': ['60 per minute'], 'headers_enabled': True, 'strategy': 'moving-window', 'storage_uri': 'memory://', 'storage_options': {}, 'in_memory_fallback': ['60 per minute'], }), } class SWHFlask(Flask): """SWH's flask application. """ response_class = SWHMultiResponse app = SWHFlask(__name__) app.add_template_filter(urlize_api_links) app.add_template_filter(urlize_header_links) app.add_template_filter(safe_docstring_display) app.add_template_filter(revision_id_from_url) app.add_template_filter(highlight_source) def read_config(config_file): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict""" conf = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(conf, 'log_dir') conf['storage'] = get_storage(**conf['storage']) return conf def load_controllers(): """Load the controllers for the application. """ from swh.web.ui import views, apidoc # flake8: noqa def rules(): """Returns rules from the application in dictionary form. Beware, must be called after swh.web.ui.main.load_controllers funcall. Returns: Generator of application's rules. """ for rule in app.url_map._rules: yield {'rule': rule.rule, 'methods': rule.methods, 'endpoint': rule.endpoint} def storage(): """Return the current application's storage. """ return app.config['conf']['storage'] def prepare_limiter(): """Prepare Flask Limiter from configuration and App configuration""" if hasattr(app, 'limiter'): return - global_limits = app.config['conf']['limiter']['global_limits'] + shared_limits = app.config['conf']['limiter'].pop('shared_limits', {}) + for name, shared_limit in shared_limits.items(): + if not shared_limit.get('exempted_networks'): + shared_limit['exempt_when'] = lambda: False + continue + + networks = [ipaddress.ip_network(network) + for network in shared_limit['exempted_networks']] + + def exempt(exempted=networks): + remote_address = ipaddress.ip_address(get_remote_address()) + return any(remote_address in network for network in exempted) + + shared_limit['exempt_when'] = exempt + limiter = Limiter( app, key_func=get_remote_address, **app.config['conf']['limiter'] ) app.limiter = limiter - for key in sorted(app.view_functions): - if key.startswith('api_'): - view_func = app.view_functions[key] - app.view_functions[key] = limiter.shared_limit( - ','.join(global_limits), - 'swh_api', - key_func=get_remote_address, - )(view_func) + for view_name in sorted(app.view_functions): + for limit_name, shared_limit in shared_limits.items(): + if view_name.startswith(shared_limit['prefix']): + view_func = app.view_functions[view_name] + app.view_functions[view_name] = limiter.shared_limit( + ','.join(shared_limit['limits']), + limit_name, + key_func=get_remote_address, + exempt_when=shared_limit['exempt_when'], + )(view_func) def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration. Note: This function is called on a per-request basis so beware the side effects here! """ if 'conf' not in app.config: load_controllers() config_path = '/etc/softwareheritage/webapp/webapp.yml' conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf prepare_limiter() logging.basicConfig(filename=os.path.join(conf['log_dir'], 'web-ui.log'), level=logging.INFO) return app(environ, start_response) def run_debug_from(config_path, verbose=False): """Run the api's server in dev mode. Note: This is called only once (contrast with the production mode in run_from_webserver function) Args: conf is a dictionary of keywords: - 'db_url' the db url's access (through psycopg2 format) - 'content_storage_dir' revisions/directories/contents storage on disk - 'host' to override the default 127.0.0.1 to open or not the server to the world - 'port' to override the default of 5004 (from the underlying layer: flask) - 'debug' activate the verbose logs - 'secret_key' the flask secret key Returns: Never """ load_controllers() conf = read_config(config_path) app.secret_key = conf['secret_key'] app.config['conf'] = conf host = conf.get('host', '127.0.0.1') port = conf.get('port') debug = conf.get('debug') prepare_limiter() log_file = os.path.join(conf['log_dir'], 'web-ui.log') logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO, handlers=[logging.FileHandler(log_file), logging.StreamHandler()]) app.run(host=host, port=port, debug=debug) diff --git a/swh/web/ui/tests/test_app.py b/swh/web/ui/tests/test_app.py index e6a89d9c..f39531a2 100644 --- a/swh/web/ui/tests/test_app.py +++ b/swh/web/ui/tests/test_app.py @@ -1,102 +1,108 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information # Functions defined here are NOT DESIGNED FOR PRODUCTION import unittest from swh.storage.api.client import RemoteStorage as Storage from swh.web.ui import main from flask_testing import TestCase # Because the Storage's __init__ function does side effect at startup... class RemoteStorageAdapter(Storage): def __init__(self, base_url): self.base_url = base_url def _init_mock_storage(base_url='https://somewhere.org:4321'): """Instanciate a remote storage whose goal is to be mocked in a test context. NOT FOR PRODUCTION Returns: An instance of swh.storage.api.client.RemoteStorage destined to be mocked (it does not do any rest call) """ return RemoteStorageAdapter(base_url) # destined to be used as mock def create_app(base_url='https://somewhere.org:4321'): """Function to initiate a flask app with storage designed to be mocked. Returns: Tuple: - app test client (for testing api, client decorator from flask) - application's full configuration - the storage instance to stub and mock - the main app without any decoration NOT FOR PRODUCTION """ storage = _init_mock_storage(base_url) # inject the mock data conf = { 'storage': storage, 'max_log_revs': 25, 'limiter': { - 'global_limits': ['10000 per hour'], 'headers_enabled': True, 'strategy': 'moving-window', 'storage_uri': 'memory://', 'storage_options': {}, - 'in_memory_fallback': ['10000 per hour'], + 'in_memory_fallback': ['1 per hour'], + 'shared_limits': { + 'swh_api': { + 'prefix': 'api_', + 'limits': ['1 per hour'], + 'exempted_networks': ['127.0.0.0/8'], + }, + }, }, } main.app.config.update({'conf': conf}) if not main.app.config['TESTING']: # HACK: install controllers only once! main.app.config['TESTING'] = True main.load_controllers() main.prepare_limiter() return main.app.test_client(), main.app.config, storage, main.app class SWHApiTestCase(unittest.TestCase): """Testing API class. """ @classmethod def setUpClass(cls): cls.app, cls.app_config, cls.storage, _ = create_app() cls.maxDiff = None class SWHViewTestCase(TestCase): """Testing view class. cf. http://pythonhosted.org/Flask-Testing/ """ # This inhibits template rendering # render_templates = False def create_app(self): """Initialize a Flask-Testing application instance to test view without template rendering """ _, _, _, appToDecorate = create_app() return appToDecorate class SWHApidocTestCase(SWHViewTestCase, SWHApiTestCase): """Testing APIDoc class. """ diff --git a/version.txt b/version.txt index 2a9c8e87..f095a2ae 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.82-0-g06d1446 \ No newline at end of file +v0.0.83-0-g3471862 \ No newline at end of file