No OneTemporary
Actions

Size

13 KB

Subscribers

None

View Options

	diff --git a/Makefile.local b/Makefile.local
	index c26b31d0..70f7896f 100644
	--- a/Makefile.local
	+++ b/Makefile.local
	@@ -1,7 +1,10 @@
	TOOL=pandoc

	run: # works with the default ~/.config/swh/webapp.yml file
	cd swh/web && python3 manage.py runserver

	+run-prod:
	+ gunicorn3 swh.web.wsgi
	+
	doc:
	cd swh/web/api/templates/includes/ && pandoc -o apidoc-header.html apidoc-header.md
	diff --git a/resources/test/webapp.yml b/resources/test/webapp.yml
	index 388b7c9b..892cc379 100644
	--- a/resources/test/webapp.yml
	+++ b/resources/test/webapp.yml
	@@ -1,21 +1,29 @@
	storage:
	cls: remote
	args:
	url: http://localhost:5002/

	# where to log information
	log_dir: /tmp/swh/web-ui/log

	# for dev only
	debug: true

	# current server (0.0.0.0 for world opening)
	host: 127.0.0.1

	# its port
	port: 5004

	# Max revisions shown in a log
	max_log_revs: 25

	-limiter_rate: '1000/min'
	+limiters:
	+ cache_uri: http://127.0.0.1:11211
	+ limits:
	+ swh_api:
	+ limiter_rate: 1/m
	+ exempted_networks:
	+ - 127.0.0.0/8
	+ another_api:
	+ limiter_rate: 2/m
	diff --git a/swh/web/common/throttling.py b/swh/web/common/throttling.py
	index b876e9a7..0b60d9cc 100644
	--- a/swh/web/common/throttling.py
	+++ b/swh/web/common/throttling.py
	@@ -1,86 +1,86 @@
	# Copyright (C) 2017 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import ipaddress

	from rest_framework.throttling import ScopedRateThrottle

	from swh.web.config import get_config


	class SwhWebRateThrottle(ScopedRateThrottle):
	"""Custom request rate limiter for DRF enabling to exempt
	specific networks specified in swh-web configuration.

	Requests are grouped into scopes. It enables to apply different
	requests rate limiting based on the scope name.

	To associate a scope to requests, one must add a 'throttle_scope'
	attribute when using a class based view, or call the 'throttle_scope'
	decorator when using a function based view. By default, requests
	do not have an associated scope and are not rate limited.

	For instance, the following YAML configuration section sets a rate of
	60 requests per minute for the 'swh_api' scope while exempting those
	comming from the 127.0.0.0/8 ip network.

	limiters:
	swh_api:
	limiter_rate: 60/min
	exempted_networks:
	- 127.0.0.0/8
	"""

	scope = None

	def __init__(self):
	super().__init__()
	self.exempted_networks = None
	- limiters = get_config()['limiters']
	+ limiters = get_config()['limiters']['limits']
	if self.scope in limiters:
	networks = limiters[self.scope].get('exempted_networks')
	if networks:
	self.exempted_networks = [ipaddress.ip_network(network)
	for network in networks]

	def allow_request(self, request, view):
	# class based view case
	if not self.scope:
	request_allowed = \
	super(SwhWebRateThrottle, self).allow_request(request, view)
	# function based view case
	else:
	self.rate = self.get_rate()
	self.num_requests, self.duration = self.parse_rate(self.rate)
	request_allowed = \
	super(ScopedRateThrottle, self).allow_request(request, view)

	if self.exempted_networks:
	remote_address = ipaddress.ip_address(self.get_ident(request))
	return any(remote_address in network
	for network in self.exempted_networks) or \
	request_allowed

	return request_allowed


	def throttle_scope(scope):
	"""Decorator that allows the throttle scope of a DRF
	function based view to be set:

	@api_view(['GET', ])
	@throttle_scope('scope')
	def view(request):
	...
	"""
	def decorator(func):
	SwhScopeRateThrottle = type(
	'CustomScopeRateThrottle',
	(SwhWebRateThrottle,),
	{'scope': scope}
	)
	func.throttle_classes = (SwhScopeRateThrottle, )
	return func
	return decorator
	diff --git a/swh/web/config.py b/swh/web/config.py
	index 3ee711d6..8b37ef15 100644
	--- a/swh/web/config.py
	+++ b/swh/web/config.py
	@@ -1,50 +1,53 @@
	# Copyright (C) 2017 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	from swh.core import config
	from swh.storage import get_storage

	DEFAULT_CONFIG = {
	'storage': ('dict', {
	'cls': 'remote',
	'args': {
	'url': 'http://127.0.0.1:5002/',
	},
	}),
	'log_dir': ('string', '/tmp/swh/log'),
	'debug': ('bool', False),
	'host': ('string', '127.0.0.1'),
	'port': ('int', 8000),
	'secret_key': ('string', 'development key'),
	'limiters': ('dict', {
	'swh_api': {
	- 'limiter_rate': '60/min',
	- 'exempted_networks': ['127.0.0.0/8']
	+ 'cache_uri': None,
	+ 'limits': [{
	+ 'limiter_rate': '60/min',
	+ 'exempted_networks': ['127.0.0.0/8']
	+ }]
	}
	})
	}

	swhweb_config = None


	def get_config(config_file='webapp'):
	"""Read the configuration file `config_file`, update the app with
	parameters (secret_key, conf) and return the parsed configuration as a
	dict. If no configuration file is provided, return a default
	configuration."""

	global swhweb_config
	if not swhweb_config:
	swhweb_config = config.load_named_config(config_file, DEFAULT_CONFIG)
	config.prepare_folders(swhweb_config, 'log_dir')
	swhweb_config['storage'] = get_storage(**swhweb_config['storage'])
	return swhweb_config


	def storage():
	"""Return the current application's SWH storage.

	"""
	return get_config()['storage']
	diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
	index de0c58ff..a467dce0 100644
	--- a/swh/web/settings/common.py
	+++ b/swh/web/settings/common.py
	@@ -1,185 +1,187 @@
	# Copyright (C) 2017 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	"""
	Django settings for swhweb project.

	Generated by 'django-admin startproject' using Django 1.11.3.

	For more information on this file, see
	https://docs.djangoproject.com/en/1.11/topics/settings/

	For the full list of settings and their values, see
	https://docs.djangoproject.com/en/1.11/ref/settings/
	"""

	import os

	from swh.web.config import get_config

	swh_web_config = get_config()

	# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
	PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))

	# Quick-start development settings - unsuitable for production
	# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/

	# SECURITY WARNING: keep the secret key used in production secret!
	SECRET_KEY = swh_web_config['secret_key']

	# SECURITY WARNING: don't run with debug turned on in production!
	DEBUG = swh_web_config['debug']

	ALLOWED_HOSTS = ['127.0.0.1', 'localhost', 'testserver']

	# Application definition

	INSTALLED_APPS = [
	'django.contrib.admin',
	'django.contrib.auth',
	'django.contrib.contenttypes',
	'django.contrib.sessions',
	'django.contrib.messages',
	'django.contrib.staticfiles',
	'rest_framework',
	'swh.web.api'
	]

	MIDDLEWARE = [
	'django.middleware.security.SecurityMiddleware',
	'django.contrib.sessions.middleware.SessionMiddleware',
	'django.middleware.common.CommonMiddleware',
	'django.middleware.csrf.CsrfViewMiddleware',
	'django.contrib.auth.middleware.AuthenticationMiddleware',
	'django.contrib.messages.middleware.MessageMiddleware',
	'django.middleware.clickjacking.XFrameOptionsMiddleware',
	]

	ROOT_URLCONF = 'swh.web.urls'

	TEMPLATES = [
	{
	'BACKEND': 'django.template.backends.django.DjangoTemplates',
	'DIRS': [],
	'APP_DIRS': True,
	'OPTIONS': {
	'context_processors': [
	'django.template.context_processors.debug',
	'django.template.context_processors.request',
	'django.contrib.auth.context_processors.auth',
	'django.contrib.messages.context_processors.messages',
	],
	},
	},
	]

	WSGI_APPLICATION = 'swh.web.wsgi.application'


	# Database
	# https://docs.djangoproject.com/en/1.11/ref/settings/#databases

	DATABASES = {
	'default': {
	'ENGINE': 'django.db.backends.sqlite3',
	'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'),
	}
	}

	# Password validation
	# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators

	AUTH_PASSWORD_VALIDATORS = [
	{
	'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa
	},
	{
	'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa
	},
	{
	'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa
	},
	{
	'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa
	},
	]


	# Internationalization
	# https://docs.djangoproject.com/en/1.11/topics/i18n/

	LANGUAGE_CODE = 'en-us'

	TIME_ZONE = 'UTC'

	USE_I18N = True

	USE_L10N = True

	USE_TZ = True

	# Static files (CSS, JavaScript, Images)
	# https://docs.djangoproject.com/en/1.11/howto/static-files/

	STATIC_URL = '/static/'
	STATICFILES_DIRS = [
	os.path.join(PROJECT_DIR, "../static")
	]

	INTERNAL_IPS = ['127.0.0.1']

	throttle_rates = {}

	-for limiter_scope, limiter_conf in swh_web_config['limiters'].items():
	- throttle_rates[limiter_scope] = None if DEBUG else limiter_conf['limiter_rate'] # noqa
	+limiters = swh_web_config['limiters']
	+for limiter_scope, limiter_conf in limiters['limits'].items():
	+ limiter_rate = None if DEBUG else limiter_conf['limiter_rate']
	+ throttle_rates[limiter_scope] = limiter_rate

	REST_FRAMEWORK = {
	'DEFAULT_RENDERER_CLASSES': (
	'rest_framework.renderers.JSONRenderer',
	'swh.web.api.renderers.YAMLRenderer',
	'rest_framework.renderers.TemplateHTMLRenderer'
	),
	'DEFAULT_THROTTLE_CLASSES': (
	'swh.web.common.throttling.SwhWebRateThrottle',
	),
	'DEFAULT_THROTTLE_RATES': throttle_rates
	}

	LOGGING = {
	'version': 1,
	'disable_existing_loggers': False,
	'filters': {
	'require_debug_false': {
	'()': 'django.utils.log.RequireDebugFalse',
	},
	'require_debug_true': {
	'()': 'django.utils.log.RequireDebugTrue',
	},
	},
	'handlers': {
	'console': {
	'level': 'INFO',
	'filters': ['require_debug_true'],
	'class': 'logging.StreamHandler',
	},
	'file': {
	'level': 'DEBUG',
	'filters': ['require_debug_false'],
	'class': 'logging.FileHandler',
	'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'),
	},
	},
	'loggers': {
	'django': {
	'handlers': ['console', 'file'],
	'level': 'DEBUG',
	'propagate': True,
	},
	},
	}

	SILENCED_SYSTEM_CHECKS = ['1_7.W001']
	diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py
	index 0db60fc1..8aa109ec 100644
	--- a/swh/web/settings/production.py
	+++ b/swh/web/settings/production.py
	@@ -1,14 +1,15 @@
	# Copyright (C) 2017 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information


	from .common import * # noqa
	+from .common import swh_web_config

	CACHES = {
	'default': {
	'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
	- 'LOCATION': '127.0.0.1:11211',
	+ 'LOCATION': swh_web_config['limiters']['cache_uri'],
	}
	}
	diff --git a/swh/web/tests/__init__.py b/swh/web/tests/__init__.py
	index 419b42ff..15245e57 100644
	--- a/swh/web/tests/__init__.py
	+++ b/swh/web/tests/__init__.py
	@@ -1,34 +1,37 @@
	# Copyright (C) 2017 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	import os
	import django

	from swh.web.config import get_config

	swh_web_config = get_config()

	swh_web_config['debug'] = False
	swh_web_config['secret_key'] = 'test'

	swh_web_config['limiters'] = {
	- 'swh_api': {
	- 'limiter_rate': '60/min',
	- 'exempted_networks': ['127.0.0.0/8']
	- },
	- 'scope1': {
	- 'limiter_rate': '3/min'
	- },
	- 'scope2': {
	- 'limiter_rate': '5/min',
	- 'exempted_networks': ['127.0.0.0/8']
	+ 'cache_uri': None,
	+ 'limits': {
	+ 'swh_api': {
	+ 'limiter_rate': '60/min',
	+ 'exempted_networks': ['127.0.0.0/8']
	+ },
	+ 'scope1': {
	+ 'limiter_rate': '3/min'
	+ },
	+ 'scope2': {
	+ 'limiter_rate': '5/min',
	+ 'exempted_networks': ['127.0.0.0/8']
	+ }
	}
	}

	os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.development")
	django.setup()

	scope1_limiter_rate = 3
	scope2_limiter_rate = 5

File Metadata

Mime Type: text/x-diff
Expires: Sat, Jun 21, 7:38 PM (3 w, 52 m ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3276683

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions