diff --git a/setup.py b/setup.py index d03de431..234c7df5 100755 --- a/setup.py +++ b/setup.py @@ -1,32 +1,32 @@ #!/usr/bin/env python3 from setuptools import setup def parse_requirements(): requirements = [] for reqf in ('requirements.txt', 'requirements-swh.txt'): with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.web', description='Software Heritage Web UI', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DWUI/', - packages=['swh.web', 'swh.web.api', + packages=['swh.web', 'swh.web.common', 'swh.web.api', 'swh.web.api.views', 'swh.web.api.tests', 'swh.web.api.templatetags'], scripts=[], install_requires=parse_requirements(), setup_requires=['vcversioner'], vcversioner={}, include_package_data=True, ) diff --git a/swh/web/api/apiurls.py b/swh/web/api/apiurls.py index da7bbe98..8d8e02a0 100644 --- a/swh/web/api/apiurls.py +++ b/swh/web/api/apiurls.py @@ -1,129 +1,132 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re from django.conf.urls import url from rest_framework.decorators import api_view +from swh.web.common.throttling import throttle_scope + class APIUrls(object): """ Class to manage API documentation URLs. - Indexes all routes documented using apidoc's decorators. - Tracks endpoint/request processing method relationships for use in generating related urls in API documentation """ apidoc_routes = {} method_endpoints = {} urlpatterns = [] @classmethod def get_app_endpoints(cls): return cls.apidoc_routes @classmethod def get_method_endpoints(cls, f): if f.__name__ not in cls.method_endpoints: cls.method_endpoints[f.__name__] = cls.group_routes_by_method(f) return cls.method_endpoints[f.__name__] @classmethod def group_routes_by_method(cls, f): """ Group URL endpoints according to their processing method. Returns: A dict where keys are the processing method names, and values are the routes that are bound to the key method. """ rules = [] for urlp in cls.urlpatterns: endpoint = urlp.callback.__name__ if endpoint != f.__name__: continue method_names = urlp.callback.http_method_names url_rule = urlp.regex.pattern.replace('^', '/').replace('$', '') url_rule_params = re.findall('\([^)]+\)', url_rule) for param in url_rule_params: param_name = re.findall('<(.*)>', param) param_name = param_name[0] if len(param_name) > 0 else None if param_name and hasattr(f, 'doc_data'): param_index = \ next(i for (i, d) in enumerate(f.doc_data['args']) if d['name'] == param_name) if param_index is not None: url_rule = url_rule.replace( param, '<' + f.doc_data['args'][param_index]['name'] + ': ' + f.doc_data['args'][param_index]['type'] + '>') rule_dict = {'rule': '/api' + url_rule, 'name': urlp.name, 'methods': {method.upper() for method in method_names} } rules.append(rule_dict) return rules @classmethod def index_add_route(cls, route, docstring, **kwargs): """ Add a route to the self-documenting API reference """ route_view_name = route[1:-1].replace('/', '-') if route not in cls.apidoc_routes: d = {'docstring': docstring, 'route_view_name': route_view_name} for k, v in kwargs.items(): d[k] = v cls.apidoc_routes[route] = d @classmethod def index_add_url_pattern(cls, url_pattern, view, view_name): cls.urlpatterns.append(url(url_pattern, view, name=view_name)) @classmethod def get_url_patterns(cls): return cls.urlpatterns class api_route(object): # noqa: N801 """ Decorator to ease the registration of an API endpoint using the Django REST Framework. Args: url_pattern: the url pattern used by DRF to identify the API route view_name: the name of the API view associated to the route used to reverse the url methods: array of HTTP methods supported by the API route """ def __init__(self, url_pattern=None, view_name=None, methods=['GET', 'HEAD'], api_version='1'): super().__init__() self.url_pattern = '^' + api_version + url_pattern + '$' self.view_name = view_name self.methods = methods def __call__(self, f): # create a DRF view from the wrapped function @api_view(self.methods) + @throttle_scope('swh_api') def api_view_f(*args, **kwargs): return f(*args, **kwargs) # small hacks for correctly generating API endpoints index doc api_view_f.__name__ = f.__name__ api_view_f.http_method_names = self.methods # register the route and its view in the endpoints index APIUrls.index_add_url_pattern(self.url_pattern, api_view_f, self.view_name) return f diff --git a/swh/web/common/__init__.py b/swh/web/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/swh/web/common/tests/test_throttling.py b/swh/web/common/tests/test_throttling.py new file mode 100644 index 00000000..07afe03c --- /dev/null +++ b/swh/web/common/tests/test_throttling.py @@ -0,0 +1,62 @@ +# Copyright (C) 2017 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from throttling_test_settings import ( + scope1_limiter_rate, scope2_limiter_rate +) + +from django.test import TestCase +from django.core.cache import cache + +from rest_framework.views import APIView +from rest_framework.response import Response +from rest_framework.test import APIRequestFactory +from rest_framework.decorators import api_view + +from swh.web.common.throttling import ( + SwhWebRateThrottle, throttle_scope +) + + +class MockView(APIView): + throttle_classes = (SwhWebRateThrottle,) + throttle_scope = 'scope1' + + def get(self, request): + return Response('foo') + + +@api_view(['GET', ]) +@throttle_scope('scope2') +def mock_view(request): + return Response('bar') + + +class ThrottlingTests(TestCase): + def setUp(self): + """ + Reset the cache so that no throttles will be active + """ + cache.clear() + self.factory = APIRequestFactory() + + def test_scope1_requests_are_throttled(self): + """ + Ensure request rate is limited in scope1 + """ + request = self.factory.get('/') + for dummy in range(scope1_limiter_rate+1): + response = MockView.as_view()(request) + assert response.status_code == 429 + + def test_scope2_requests_are_throttled(self): + """ + Ensure request rate is not limited in scope2 as + requests coming from localhost are exempted from rate limit. + """ + request = self.factory.get('/') + for dummy in range(scope2_limiter_rate+1): + response = mock_view(request) + assert response.status_code == 200 diff --git a/swh/web/common/tests/throttling_test_settings.py b/swh/web/common/tests/throttling_test_settings.py new file mode 100644 index 00000000..3f578119 --- /dev/null +++ b/swh/web/common/tests/throttling_test_settings.py @@ -0,0 +1,29 @@ +# Copyright (C) 2017 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import os +import django + +from swh.web.config import get_config + +swh_web_config = get_config() + +swh_web_config['debug'] = False + +swh_web_config['limiters'] = { + 'scope1': { + 'limiter_rate': '3/min' + }, + 'scope2': { + 'limiter_rate': '5/min', + 'exempted_networks': ['127.0.0.0/8'] + } +} + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings") +django.setup() + +scope1_limiter_rate = 3 +scope2_limiter_rate = 5 diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py new file mode 100644 index 00000000..b876e9a7 --- /dev/null +++ b/swh/web/common/throttling.py @@ -0,0 +1,86 @@ +# Copyright (C) 2017 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import ipaddress + +from rest_framework.throttling import ScopedRateThrottle + +from swh.web.config import get_config + + +class SwhWebRateThrottle(ScopedRateThrottle): + """Custom request rate limiter for DRF enabling to exempt + specific networks specified in swh-web configuration. + + Requests are grouped into scopes. It enables to apply different + requests rate limiting based on the scope name. + + To associate a scope to requests, one must add a 'throttle_scope' + attribute when using a class based view, or call the 'throttle_scope' + decorator when using a function based view. By default, requests + do not have an associated scope and are not rate limited. + + For instance, the following YAML configuration section sets a rate of + 60 requests per minute for the 'swh_api' scope while exempting those + comming from the 127.0.0.0/8 ip network. + + limiters: + swh_api: + limiter_rate: 60/min + exempted_networks: + - 127.0.0.0/8 + """ + + scope = None + + def __init__(self): + super().__init__() + self.exempted_networks = None + limiters = get_config()['limiters'] + if self.scope in limiters: + networks = limiters[self.scope].get('exempted_networks') + if networks: + self.exempted_networks = [ipaddress.ip_network(network) + for network in networks] + + def allow_request(self, request, view): + # class based view case + if not self.scope: + request_allowed = \ + super(SwhWebRateThrottle, self).allow_request(request, view) + # function based view case + else: + self.rate = self.get_rate() + self.num_requests, self.duration = self.parse_rate(self.rate) + request_allowed = \ + super(ScopedRateThrottle, self).allow_request(request, view) + + if self.exempted_networks: + remote_address = ipaddress.ip_address(self.get_ident(request)) + return any(remote_address in network + for network in self.exempted_networks) or \ + request_allowed + + return request_allowed + + +def throttle_scope(scope): + """Decorator that allows the throttle scope of a DRF + function based view to be set: + + @api_view(['GET', ]) + @throttle_scope('scope') + def view(request): + ... + """ + def decorator(func): + SwhScopeRateThrottle = type( + 'CustomScopeRateThrottle', + (SwhWebRateThrottle,), + {'scope': scope} + ) + func.throttle_classes = (SwhScopeRateThrottle, ) + return func + return decorator diff --git a/swh/web/config.py b/swh/web/config.py index 58980cc0..a82c65ef 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,45 +1,50 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import config from swh.storage import get_storage DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', True), 'host': ('string', '127.0.0.1'), 'port': ('int', 8000), 'secret_key': ('string', 'development key'), - 'limiter_rate': ('string', '60/min') + 'limiters': ('dict', { + 'swh_api': { + 'limiter_rate': '60/min', + 'exempted_networks': ['127.0.0.0/8'] + } + }) } swhweb_config = None def get_config(config_file=None): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration.""" global swhweb_config if not swhweb_config or config_file: swhweb_config = config.read(config_file, DEFAULT_CONFIG) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) return swhweb_config def storage(): """Return the current application's SWH storage. """ return get_config()['storage'] diff --git a/swh/web/settings.py b/swh/web/settings.py index d52dbbe2..cd352c04 100644 --- a/swh/web/settings.py +++ b/swh/web/settings.py @@ -1,180 +1,184 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django settings for swhweb project. Generated by 'django-admin startproject' using Django 1.11.3. For more information on this file, see https://docs.djangoproject.com/en/1.11/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost', 'testserver'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.api' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', ], }, }, ] WSGI_APPLICATION = 'swh.web.wsgi.application' # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'), } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "static") ] INTERNAL_IPS = ['127.0.0.1'] +throttle_rates = {} + +for limiter_scope, limiter_conf in swh_web_config['limiters'].items(): + throttle_rates[limiter_scope] = None if DEBUG else limiter_conf['limiter_rate'] # noqa + + REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( - 'rest_framework.throttling.AnonRateThrottle', + 'swh.web.common.throttling.SwhWebRateThrottle', ), - 'DEFAULT_THROTTLE_RATES': { - 'anon': None if DEBUG else swh_web_config['limiter_rate'], - } + 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'handlers': { 'console': { 'level': 'INFO', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', }, 'file': { 'level': 'DEBUG', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), }, }, 'loggers': { 'django': { 'handlers': ['console', 'file'], 'level': 'DEBUG', 'propagate': True, }, }, }