diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py index 0b60d9cc0..edafdb91f 100644 --- a/swh/web/common/throttling.py +++ b/swh/web/common/throttling.py @@ -1,86 +1,88 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import ipaddress from rest_framework.throttling import ScopedRateThrottle from swh.web.config import get_config class SwhWebRateThrottle(ScopedRateThrottle): """Custom request rate limiter for DRF enabling to exempt specific networks specified in swh-web configuration. Requests are grouped into scopes. It enables to apply different requests rate limiting based on the scope name. To associate a scope to requests, one must add a 'throttle_scope' attribute when using a class based view, or call the 'throttle_scope' decorator when using a function based view. By default, requests do not have an associated scope and are not rate limited. For instance, the following YAML configuration section sets a rate of 60 requests per minute for the 'swh_api' scope while exempting those comming from the 127.0.0.0/8 ip network. - limiters: - swh_api: - limiter_rate: 60/min - exempted_networks: - - 127.0.0.0/8 + throttling: + scopes: + swh_api: + limiter_rate: 60/min + exempted_networks: + - 127.0.0.0/8 """ scope = None def __init__(self): super().__init__() self.exempted_networks = None - limiters = get_config()['limiters']['limits'] - if self.scope in limiters: - networks = limiters[self.scope].get('exempted_networks') + scopes = get_config()['throttling']['scopes'] + scope = scopes.get(self.scope) + if scope: + networks = scope.get('exempted_networks') if networks: self.exempted_networks = [ipaddress.ip_network(network) for network in networks] def allow_request(self, request, view): # class based view case if not self.scope: request_allowed = \ super(SwhWebRateThrottle, self).allow_request(request, view) # function based view case else: self.rate = self.get_rate() self.num_requests, self.duration = self.parse_rate(self.rate) request_allowed = \ super(ScopedRateThrottle, self).allow_request(request, view) if self.exempted_networks: remote_address = ipaddress.ip_address(self.get_ident(request)) return any(remote_address in network for network in self.exempted_networks) or \ request_allowed return request_allowed def throttle_scope(scope): """Decorator that allows the throttle scope of a DRF function based view to be set: @api_view(['GET', ]) @throttle_scope('scope') def view(request): ... """ def decorator(func): SwhScopeRateThrottle = type( 'CustomScopeRateThrottle', (SwhWebRateThrottle,), {'scope': scope} ) func.throttle_classes = (SwhScopeRateThrottle, ) return func return decorator diff --git a/swh/web/config.py b/swh/web/config.py index 6793ee81d..62c5f52bd 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,54 +1,54 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import config from swh.storage import get_storage DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'host': ('string', '127.0.0.1'), # development property 'port': ('int', 5003), # development property 'secret_key': ('string', 'development key'), - 'limiters': ('dict', { + 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None - 'limits': { + 'scopes': { 'swh_api': [{ 'limiter_rate': '120/h', 'exempted_networks': ['127.0.0.0/8'] }] } }) } swhweb_config = None def get_config(config_file='webapp'): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration.""" global swhweb_config if not swhweb_config: swhweb_config = config.load_named_config(config_file, DEFAULT_CONFIG) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) return swhweb_config def storage(): """Return the current application's SWH storage. """ return get_config()['storage'] diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index a467dce08..57e67d007 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,187 +1,187 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django settings for swhweb project. Generated by 'django-admin startproject' using Django 1.11.3. For more information on this file, see https://docs.djangoproject.com/en/1.11/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost', 'testserver'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.api' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', ], }, }, ] WSGI_APPLICATION = 'swh.web.wsgi.application' # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'), } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "../static") ] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} -limiters = swh_web_config['limiters'] -for limiter_scope, limiter_conf in limiters['limits'].items(): +throttling = swh_web_config['throttling'] +for limiter_scope, limiter_conf in throttling['scopes'].items(): limiter_rate = None if DEBUG else limiter_conf['limiter_rate'] throttle_rates[limiter_scope] = limiter_rate REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'handlers': { 'console': { 'level': 'INFO', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', }, 'file': { 'level': 'DEBUG', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), }, }, 'loggers': { 'django': { 'handlers': ['console', 'file'], 'level': 'DEBUG', 'propagate': True, }, }, } SILENCED_SYSTEM_CHECKS = ['1_7.W001'] diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py index 8aa109ecf..3a53508c1 100644 --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -1,15 +1,15 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from .common import * # noqa from .common import swh_web_config CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': swh_web_config['limiters']['cache_uri'], + 'LOCATION': swh_web_config['throttling']['cache_uri'], } } diff --git a/swh/web/tests/__init__.py b/swh/web/tests/__init__.py index 15245e57f..4777d380b 100644 --- a/swh/web/tests/__init__.py +++ b/swh/web/tests/__init__.py @@ -1,37 +1,38 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import django from swh.web.config import get_config swh_web_config = get_config() -swh_web_config['debug'] = False -swh_web_config['secret_key'] = 'test' - -swh_web_config['limiters'] = { - 'cache_uri': None, - 'limits': { - 'swh_api': { - 'limiter_rate': '60/min', - 'exempted_networks': ['127.0.0.0/8'] - }, - 'scope1': { - 'limiter_rate': '3/min' - }, - 'scope2': { - 'limiter_rate': '5/min', - 'exempted_networks': ['127.0.0.0/8'] +swh_web_config.update({ + 'debug': False, + 'secret_key': 'test', + 'throttling': { + 'cache_uri': None, + 'scopes': { + 'swh_api': { + 'limiter_rate': '60/min', + 'exempted_networks': ['127.0.0.0/8'] + }, + 'scope1': { + 'limiter_rate': '3/min' + }, + 'scope2': { + 'limiter_rate': '5/min', + 'exempted_networks': ['127.0.0.0/8'] + } } } -} +}) os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.development") django.setup() scope1_limiter_rate = 3 scope2_limiter_rate = 5