diff --git a/Makefile.local b/Makefile.local index c26b31d0..70f7896f 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,7 +1,10 @@ TOOL=pandoc run: # works with the default ~/.config/swh/webapp.yml file cd swh/web && python3 manage.py runserver +run-prod: + gunicorn3 swh.web.wsgi + doc: cd swh/web/api/templates/includes/ && pandoc -o apidoc-header.html apidoc-header.md diff --git a/resources/test/webapp.yml b/resources/test/webapp.yml index 388b7c9b..892cc379 100644 --- a/resources/test/webapp.yml +++ b/resources/test/webapp.yml @@ -1,21 +1,29 @@ storage: cls: remote args: url: http://localhost:5002/ # where to log information log_dir: /tmp/swh/web-ui/log # for dev only debug: true # current server (0.0.0.0 for world opening) host: 127.0.0.1 # its port port: 5004 # Max revisions shown in a log max_log_revs: 25 -limiter_rate: '1000/min' +limiters: + cache_uri: http://127.0.0.1:11211 + limits: + swh_api: + limiter_rate: 1/m + exempted_networks: + - 127.0.0.0/8 + another_api: + limiter_rate: 2/m diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py index b876e9a7..0b60d9cc 100644 --- a/swh/web/common/throttling.py +++ b/swh/web/common/throttling.py @@ -1,86 +1,86 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import ipaddress from rest_framework.throttling import ScopedRateThrottle from swh.web.config import get_config class SwhWebRateThrottle(ScopedRateThrottle): """Custom request rate limiter for DRF enabling to exempt specific networks specified in swh-web configuration. Requests are grouped into scopes. It enables to apply different requests rate limiting based on the scope name. To associate a scope to requests, one must add a 'throttle_scope' attribute when using a class based view, or call the 'throttle_scope' decorator when using a function based view. By default, requests do not have an associated scope and are not rate limited. For instance, the following YAML configuration section sets a rate of 60 requests per minute for the 'swh_api' scope while exempting those comming from the 127.0.0.0/8 ip network. limiters: swh_api: limiter_rate: 60/min exempted_networks: - 127.0.0.0/8 """ scope = None def __init__(self): super().__init__() self.exempted_networks = None - limiters = get_config()['limiters'] + limiters = get_config()['limiters']['limits'] if self.scope in limiters: networks = limiters[self.scope].get('exempted_networks') if networks: self.exempted_networks = [ipaddress.ip_network(network) for network in networks] def allow_request(self, request, view): # class based view case if not self.scope: request_allowed = \ super(SwhWebRateThrottle, self).allow_request(request, view) # function based view case else: self.rate = self.get_rate() self.num_requests, self.duration = self.parse_rate(self.rate) request_allowed = \ super(ScopedRateThrottle, self).allow_request(request, view) if self.exempted_networks: remote_address = ipaddress.ip_address(self.get_ident(request)) return any(remote_address in network for network in self.exempted_networks) or \ request_allowed return request_allowed def throttle_scope(scope): """Decorator that allows the throttle scope of a DRF function based view to be set: @api_view(['GET', ]) @throttle_scope('scope') def view(request): ... """ def decorator(func): SwhScopeRateThrottle = type( 'CustomScopeRateThrottle', (SwhWebRateThrottle,), {'scope': scope} ) func.throttle_classes = (SwhScopeRateThrottle, ) return func return decorator diff --git a/swh/web/config.py b/swh/web/config.py index 3ee711d6..8b37ef15 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,50 +1,53 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import config from swh.storage import get_storage DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', }, }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'host': ('string', '127.0.0.1'), 'port': ('int', 8000), 'secret_key': ('string', 'development key'), 'limiters': ('dict', { 'swh_api': { - 'limiter_rate': '60/min', - 'exempted_networks': ['127.0.0.0/8'] + 'cache_uri': None, + 'limits': [{ + 'limiter_rate': '60/min', + 'exempted_networks': ['127.0.0.0/8'] + }] } }) } swhweb_config = None def get_config(config_file='webapp'): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration.""" global swhweb_config if not swhweb_config: swhweb_config = config.load_named_config(config_file, DEFAULT_CONFIG) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) return swhweb_config def storage(): """Return the current application's SWH storage. """ return get_config()['storage'] diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index de0c58ff..a467dce0 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,185 +1,187 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django settings for swhweb project. Generated by 'django-admin startproject' using Django 1.11.3. For more information on this file, see https://docs.djangoproject.com/en/1.11/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.11/ref/settings/ """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost', 'testserver'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.api' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', ], }, }, ] WSGI_APPLICATION = 'swh.web.wsgi.application' # Database # https://docs.djangoproject.com/en/1.11/ref/settings/#databases DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'), } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "../static") ] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} -for limiter_scope, limiter_conf in swh_web_config['limiters'].items(): - throttle_rates[limiter_scope] = None if DEBUG else limiter_conf['limiter_rate'] # noqa +limiters = swh_web_config['limiters'] +for limiter_scope, limiter_conf in limiters['limits'].items(): + limiter_rate = None if DEBUG else limiter_conf['limiter_rate'] + throttle_rates[limiter_scope] = limiter_rate REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'handlers': { 'console': { 'level': 'INFO', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', }, 'file': { 'level': 'DEBUG', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), }, }, 'loggers': { 'django': { 'handlers': ['console', 'file'], 'level': 'DEBUG', 'propagate': True, }, }, } SILENCED_SYSTEM_CHECKS = ['1_7.W001'] diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py index 0db60fc1..8aa109ec 100644 --- a/swh/web/settings/production.py +++ b/swh/web/settings/production.py @@ -1,14 +1,15 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from .common import * # noqa +from .common import swh_web_config CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': '127.0.0.1:11211', + 'LOCATION': swh_web_config['limiters']['cache_uri'], } } diff --git a/swh/web/tests/__init__.py b/swh/web/tests/__init__.py index 419b42ff..15245e57 100644 --- a/swh/web/tests/__init__.py +++ b/swh/web/tests/__init__.py @@ -1,34 +1,37 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import django from swh.web.config import get_config swh_web_config = get_config() swh_web_config['debug'] = False swh_web_config['secret_key'] = 'test' swh_web_config['limiters'] = { - 'swh_api': { - 'limiter_rate': '60/min', - 'exempted_networks': ['127.0.0.0/8'] - }, - 'scope1': { - 'limiter_rate': '3/min' - }, - 'scope2': { - 'limiter_rate': '5/min', - 'exempted_networks': ['127.0.0.0/8'] + 'cache_uri': None, + 'limits': { + 'swh_api': { + 'limiter_rate': '60/min', + 'exempted_networks': ['127.0.0.0/8'] + }, + 'scope1': { + 'limiter_rate': '3/min' + }, + 'scope2': { + 'limiter_rate': '5/min', + 'exempted_networks': ['127.0.0.0/8'] + } } } os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.development") django.setup() scope1_limiter_rate = 3 scope2_limiter_rate = 5