diff --git a/Makefile.local b/Makefile.local index b94c4cba..10ea611e 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,41 +1,41 @@ .PHONY: build-webpack-dev build-webpack-dev: npm run build-dev .PHONY: build-webpack-dev-no-verbose build-webpack-dev-no-verbose: npm run build-dev >/dev/null .PHONY: build-webpack-prod build-webpack-prod: npm run build .PHONY: run-migrations run-migrations: python3 swh/web/manage.py migrate 2>/dev/null .PHONY: run-tests-migrations run-tests-migrations: rm -f swh/web/settings/testdb.sqlite3 2>/dev/null django-admin migrate --settings=swh.web.settings.tests 2>/dev/null run-django-webpack-devserver: run-migrations - bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; npm run start-dev & cd swh/web && python3 manage.py runserver" + bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; npm run start-dev & cd swh/web && python3 manage.py runserver --nostatic" run-django-webpack-dev: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic run-django-webpack-prod: build-webpack-prod run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-django-server-dev: run-migrations python3 swh/web/manage.py runserver --nostatic run-django-server-prod: run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-gunicorn-server: run-migrations gunicorn3 -b 127.0.0.1:5004 swh.web.wsgi test: build-webpack-dev-no-verbose run-tests-migrations diff --git a/swh/web/config.py b/swh/web/config.py index e4052284..d9998139 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,128 +1,129 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core import config from swh.storage import get_storage from swh.indexer.storage import get_indexer_storage from swh.vault.api.client import RemoteVaultClient from swh.scheduler import get_scheduler DEFAULT_CONFIG = { 'allowed_hosts': ('list', []), 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', 'timeout': 10, }, }), 'indexer_storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5007/', 'timeout': 1, } }), 'vault': ('string', 'http://127.0.0.1:5005/'), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), + 'serve_assets': ('bool', False), 'host': ('string', '127.0.0.1'), 'port': ('int', 5004), 'secret_key': ('string', 'development key'), # do not display code highlighting for content > 1MB 'content_display_max_size': ('int', 1024 * 1024), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '120/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '10/h' }, 'exempted_networks': ['127.0.0.0/8'] } } }), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://localhost:5008/' } }), 'grecaptcha': ('dict', { 'validation_url': 'https://www.google.com/recaptcha/api/siteverify', 'site_key': '', 'private_key': '' }), 'production_db': ('string', '/var/lib/swh/web.sqlite3'), 'deposit': ('dict', { 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', 'private_api_user': 'swhworker', 'private_api_password': '' }) } swhweb_config = {} def get_config(config_file='webapp/webapp'): """Read the configuration file `config_file`, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration.""" if not swhweb_config: cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) swhweb_config['vault'] = RemoteVaultClient(swhweb_config['vault']) swhweb_config['indexer_storage'] = \ get_indexer_storage(**swhweb_config['indexer_storage']) swhweb_config['scheduler'] = get_scheduler(**swhweb_config['scheduler']) # noqa return swhweb_config def storage(): """Return the current application's SWH storage. """ return get_config()['storage'] def vault(): """Return the current application's SWH vault. """ return get_config()['vault'] def indexer_storage(): """Return the current application's SWH indexer storage. """ return get_config()['indexer_storage'] def scheduler(): """Return the current application's SWH scheduler. """ return get_config()['scheduler'] diff --git a/swh/web/manage.py b/swh/web/manage.py index b292a420..52277bf3 100755 --- a/swh/web/manage.py +++ b/swh/web/manage.py @@ -1,49 +1,53 @@ #!/usr/bin/env python # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os import sys from swh.web import config if __name__ == "__main__": + swh_web_config = config.get_config() + # the serving of static assets in development mode is handled + # in swh/web/urls.py, we pass the nostatic options to runserver + # in order to have gzip compression enabled. + swh_web_config['serve_assets'] = '--nostatic' in sys.argv if len(sys.argv) > 1 and sys.argv[1] == 'test': os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.tests") else: os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.development") # import root urls module for swh-web before running the django dev server # in order to ensure it will be automatically reloaded when source files # are modified (as django autoreload feature only works if the modules are # in sys.modules) try: from swh.web import urls # noqa except Exception: pass try: from django.core.management.commands.runserver import ( Command as runserver ) from django.core.management import execute_from_command_line except ImportError: # The above import may fail for some other reason. Ensure that the # issue is really that Django is missing to avoid masking other # exceptions on Python 2. try: import django # noqa except ImportError: raise ImportError( "Couldn't import Django. Are you sure it's installed and " "available on your PYTHONPATH environment variable? Did you " "forget to activate a virtual environment?" ) raise - swh_web_config = config.get_config() runserver.default_port = swh_web_config['port'] runserver.default_addr = swh_web_config['host'] execute_from_command_line(sys.argv) diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 162bfc55..8f3b5b7d 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,233 +1,233 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.common', 'swh.web.api', 'swh.web.browse', 'webpack_loader', 'django_js_reverse' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'swh.web.common.middlewares.ThrottlingHeadersMiddleware' ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. -if swh_web_config['debug']: +if swh_web_config['serve_assets']: MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware') ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [os.path.join(PROJECT_DIR, "../templates")], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', ], 'libraries': { 'swh_templatetags': 'swh.web.common.swh_templatetags', }, }, }, ] WSGI_APPLICATION = 'swh.web.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': os.path.join(PROJECT_DIR, 'db.sqlite3'), } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "../static") ] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'] throttling = swh_web_config['throttling'] for limiter_scope, limiter_conf in throttling['scopes'].items(): if 'default' in limiter_conf['limiter_rate']: throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default'] # for backward compatibility else: throttle_rates[limiter_scope] = limiter_conf['limiter_rate'] # register sub scopes specific for HTTP request types for http_request in http_requests: if http_request in limiter_conf['limiter_rate']: throttle_rates[limiter_scope + '_' + http_request.lower()] = \ limiter_conf['limiter_rate'][http_request] REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'formatters': { 'verbose': { 'format': '[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, }, 'handlers': { 'console': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', }, 'file': { 'level': 'INFO', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'verbose' }, 'null': { 'class': 'logging.NullHandler', }, }, 'loggers': { 'django': { 'handlers': ['console', 'file'], 'level': 'DEBUG' if DEBUG else 'INFO', 'propagate': True, }, 'django.request': { 'handlers': ['file'], 'level': 'DEBUG' if DEBUG else 'INFO', 'propagate': False, }, 'django.db.backends': { 'handlers': ['null'], 'propagate': False } }, } WEBPACK_LOADER = { 'DEFAULT': { 'CACHE': not DEBUG, 'BUNDLE_DIR_NAME': './', 'STATS_FILE': os.path.join(PROJECT_DIR, '../static/webpack-stats.json'), # noqa 'POLL_INTERVAL': 0.1, 'TIMEOUT': None, 'IGNORE': ['.+\.hot-update.js', '.+\.map'] } } LOGIN_URL = '/admin/login/' LOGIN_REDIRECT_URL = 'admin' diff --git a/swh/web/urls.py b/swh/web/urls.py index d2ee54a9..d30ca8a8 100644 --- a/swh/web/urls.py +++ b/swh/web/urls.py @@ -1,52 +1,59 @@ # Copyright (C) 2017-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf import settings from django.conf.urls import ( url, include, handler400, handler403, handler404, handler500 ) -from django.contrib.staticfiles.urls import staticfiles_urlpatterns from django.contrib.staticfiles.views import serve from django.shortcuts import render from django.views.generic.base import RedirectView from django_js_reverse.views import urls_js +from swh.web.browse.identifiers import swh_id_browse +from swh.web.config import get_config from swh.web.common.exc import ( swh_handle400, swh_handle403, swh_handle404, swh_handle500 ) -from swh.web.browse.identifiers import swh_id_browse +swh_web_config = get_config() favicon_view = RedirectView.as_view(url='/static/img/icons/swh-logo-32x32.png', permanent=True) def default_view(request): return render(request, "homepage.html") urlpatterns = [ url(r'^admin/', include('swh.web.admin.urls')), url(r'^favicon\.ico$', favicon_view), url(r'^api/', include('swh.web.api.urls')), url(r'^browse/', include('swh.web.browse.urls')), url(r'^$', default_view, name='swh-web-homepage'), url(r'^jsreverse/$', urls_js, name='js_reverse'), url(r'^(?Pswh:[0-9]+:[a-z]+:[0-9a-f]+.*)/$', swh_id_browse, name='browse-swh-id') ] + +# allow to serve assets through django staticfiles +# even if settings.DEBUG is False +def insecure_serve(request, path, **kwargs): + return serve(request, path, insecure=True, **kwargs) + + # enable to serve compressed assets through django development server -if settings.DEBUG: +if swh_web_config['serve_assets']: static_pattern = r'^%s(?P.*)$' % settings.STATIC_URL[1:] - urlpatterns.append(url(static_pattern, serve)) -else: - urlpatterns += staticfiles_urlpatterns() + urlpatterns.append(url(static_pattern, insecure_serve)) + handler400 = swh_handle400 # noqa handler403 = swh_handle403 # noqa handler404 = swh_handle404 # noqa handler500 = swh_handle500 # noqa