diff --git a/Makefile.local b/Makefile.local index 78c5b77a..e098abb8 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,88 +1,89 @@ TEST_DIRS := ./swh/web/tests TESTFLAGS = --hypothesis-profile=swh-web-fast TESTFULL_FLAGS = --hypothesis-profile=swh-web YARN ?= yarn yarn-install: package.json $(YARN) install .PHONY: build-webpack-dev build-webpack-dev: yarn-install $(YARN) build-dev .PHONY: build-webpack-test build-webpack-test: yarn-install $(YARN) build-test .PHONY: build-webpack-dev-no-verbose build-webpack-dev-no-verbose: yarn-install $(YARN) build-dev >/dev/null .PHONY: build-webpack-prod build-webpack-prod: yarn-install $(YARN) build .PHONY: run-migrations run-migrations: python3 swh/web/manage.py migrate --settings=swh.web.settings.development -v0 2>/dev/null python3 swh/web/manage.py createcachetable --settings=swh.web.settings.development -v0 2>/dev/null .PHONY: run-migrations-prod run-migrations-prod: django-admin migrate --settings=swh.web.settings.production -v0 2>/dev/null django-admin createcachetable --settings=swh.web.settings.production -v0 2>/dev/null .PHONY: run-migrations-test run-migrations-test: rm -f swh/web/settings/testdb.sqlite3 django-admin migrate --settings=swh.web.settings.tests -v0 2>/dev/null django-admin createcachetable --settings=swh.web.settings.tests -v0 2>/dev/null cat swh/web/tests/create_test_admin.py | django-admin shell --settings=swh.web.settings.tests .PHONY: clear-memcached clear-memcached: echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null run-django-webpack-devserver: run-migrations yarn-install bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; $(YARN) start-dev & sleep 10 && cd swh/web && python3 manage.py runserver --nostatic --settings=swh.web.settings.development" run-django-webpack-dev: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.development run-django-webpack-prod: build-webpack-prod run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-django-server-dev: run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.development run-django-server-prod: run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-gunicorn-server: run-migrations clear-memcached - gunicorn3 -b 127.0.0.1:5004 swh.web.wsgi + DJANGO_SETTINGS_MODULE=swh.web.settings.production \ + gunicorn3 -b 127.0.0.1:5004 'django.core.wsgi:get_wsgi_application()' run-django-webpack-memory-storages: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests test-full: $(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS) .PHONY: test-frontend-cmd test-frontend-cmd: build-webpack-test run-migrations-test python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress run ; pkill -P $$! ; $(YARN) run mochawesome test-frontend: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend: test-frontend-cmd test-frontend-full: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full: test-frontend-cmd .PHONY: test-frontend-ui-cmd test-frontend-ui-cmd: build-webpack-test run-migrations-test bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT; jobs -p | head -1 | xargs pkill -P' SIGINT SIGTERM ERR EXIT; python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress open" test-frontend-ui: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend-ui: test-frontend-ui-cmd test-frontend-full-ui: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full-ui: test-frontend-ui-cmd diff --git a/docs/developers-info.rst b/docs/developers-info.rst index 847886b3..39520e58 100644 --- a/docs/developers-info.rst +++ b/docs/developers-info.rst @@ -1,129 +1,127 @@ Developers Information ====================== Sample configuration -------------------- The configuration will be taken from the default configuration file: ``~/.config/swh/web/web.yml``. The following introduces a default configuration file: .. sourcecode:: yaml storage: cls: remote args: url: http://localhost:5002 debug: false throttling: cache_uri: None scopes: swh_api: limiter_rate: default: 120/h exempted_networks: - 127.0.0.0/8 Run server ---------- Either use the django manage script directly (useful in development mode as it offers various commands): .. sourcecode:: shell $ python3 -m swh.web.manage runserver or use the following shortcut: .. sourcecode:: shell $ make run Modules description ------------------- Common to all web applications ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Configuration and settings """""""""""""""""""""""""" * :mod:`swh.web.config`: holds the configuration for the web applications. * :mod:`swh.web.doc_config`: utility module used to extend the sphinx configuration when building the documentation. * :mod:`swh.web.manage`: Django management module for developers. * :mod:`swh.web.urls`: module that holds the whole URI scheme of all the web applications. - * :mod:`swh.web.wsgi`: WSGI module to use when deploying the web applications - in production. * :mod:`swh.web.settings.common`: Common Django settings * :mod:`swh.web.settings.development`: Django settings for development * :mod:`swh.web.settings.production`: Django settings for production * :mod:`swh.web.settings.tests`: Django settings for tests Common utilities """""""""""""""" * :mod:`swh.web.common.converters`: conversion module used to transform raw data to serializable ones. It is used by :mod:`swh.web.common.service`: to convert data before transmitting then to Django views. * :mod:`swh.web.common.exc`: module defining exceptions used in the web applications. * :mod:`swh.web.common.highlightjs`: utility module to ease the use of the highlightjs_ library in produced Django views. * :mod:`swh.web.common.query`: Utilities to parse data from HTTP endpoints. It is used by :mod:`swh.web.common.service`. * :mod:`swh.web.common.service`: Orchestration layer used by views module in charge of communication with :mod:`swh.storage` to retrieve information and perform conversion for the upper layer. * :mod:`swh.web.common.swh_templatetags`: Custom Django template tags library for swh. * :mod:`swh.web.common.throttling`: Custom request rate limiter to use with the `Django REST Framework `_ * :mod:`swh.web.common.urlsindex`: Utilities to help the registering of endpoints for the web applications * :mod:`swh.web.common.utils`: Utility functions used in the web applications implementation swh-web API application ^^^^^^^^^^^^^^^^^^^^^^^ * :mod:`swh.web.api.apidoc`: Utilities to document the web api for its html browsable rendering. * :mod:`swh.web.api.apiresponse`: Utility module to ease the generation of web api responses. * :mod:`swh.web.api.apiurls`: Utilities to facilitate the registration of web api endpoints. * :mod:`swh.web.api.urls`: Module that defines the whole URI scheme for the api endpoints * :mod:`swh.web.api.utils`: Utility functions used in the web api implementation. * :mod:`swh.web.api.views.content`: Implementation of API endpoints for getting information about contents. * :mod:`swh.web.api.views.directory`: Implementation of API endpoints for getting information about directories. * :mod:`swh.web.api.views.origin`: Implementation of API endpoints for getting information about origins. * :mod:`swh.web.api.views.person`: Implementation of API endpoints for getting information about persons. * :mod:`swh.web.api.views.release`: Implementation of API endpoints for getting information about releases. * :mod:`swh.web.api.views.revision`: Implementation of API endpoints for getting information about revisions. * :mod:`swh.web.api.views.snapshot`: Implementation of API endpoints for getting information about snapshots. * :mod:`swh.web.api.views.stat`: Implementation of API endpoints for getting information about archive statistics. * :mod:`swh.web.api.views.utils`: Utilities used in the web api endpoints implementation. swh-web browse application ^^^^^^^^^^^^^^^^^^^^^^^^^^ * :mod:`swh.web.browse.browseurls`: Utilities to facilitate the registration of browse endpoints. * :mod:`swh.web.browse.urls`: Module that defines the whole URI scheme for the browse endpoints. * :mod:`swh.web.browse.utils`: Utilities functions used throughout the browse endpoints implementation. * :mod:`swh.web.browse.views.content`: Implementation of endpoints for browsing contents. * :mod:`swh.web.browse.views.directory`: Implementation of endpoints for browsing directories. * :mod:`swh.web.browse.views.identifiers`: Implementation of endpoints for browsing objects through persistent identifiers. * :mod:`swh.web.browse.views.origin`: Implementation of endpoints for browsing origins. * :mod:`swh.web.browse.views.person`: Implementation of endpoints for browsing persons. * :mod:`swh.web.browse.views.release`: Implementation of endpoints for browsing releases. * :mod:`swh.web.browse.views.revision`: Implementation of endpoints for browsing revisions. * :mod:`swh.web.browse.views.snapshot`: Implementation of endpoints for browsing snapshots. .. _highlightjs: https://highlightjs.org/ diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 0cfd381f..e0fdea50 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,280 +1,279 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config['secret_key'] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config['debug'] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config['debug'] ALLOWED_HOSTS = ['127.0.0.1', 'localhost'] + swh_web_config['allowed_hosts'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework', 'swh.web.common', 'swh.web.api', 'swh.web.browse', 'webpack_loader', 'django_js_reverse' ] MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'swh.web.common.middlewares.ThrottlingHeadersMiddleware' ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. if swh_web_config['serve_assets']: MIDDLEWARE.insert(0, 'django.middleware.gzip.GZipMiddleware') ROOT_URLCONF = 'swh.web.urls' TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', 'DIRS': [os.path.join(PROJECT_DIR, "../templates")], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ 'django.template.context_processors.debug', 'django.template.context_processors.request', 'django.contrib.auth.context_processors.auth', 'django.contrib.messages.context_processors.messages', 'swh.web.common.utils.context_processor' ], 'libraries': { 'swh_templatetags': 'swh.web.common.swh_templatetags', }, }, }, ] -WSGI_APPLICATION = 'swh.web.wsgi.application' DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', 'NAME': swh_web_config['development_db'], } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa }, { 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa }, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = 'en-us' TIME_ZONE = 'UTC' USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = '/static/' STATICFILES_DIRS = [ os.path.join(PROJECT_DIR, "../static") ] INTERNAL_IPS = ['127.0.0.1'] throttle_rates = {} http_requests = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'OPTIONS', 'PATCH'] throttling = swh_web_config['throttling'] for limiter_scope, limiter_conf in throttling['scopes'].items(): if 'default' in limiter_conf['limiter_rate']: throttle_rates[limiter_scope] = limiter_conf['limiter_rate']['default'] # for backward compatibility else: throttle_rates[limiter_scope] = limiter_conf['limiter_rate'] # register sub scopes specific for HTTP request types for http_request in http_requests: if http_request in limiter_conf['limiter_rate']: throttle_rates[limiter_scope + '_' + http_request.lower()] = \ limiter_conf['limiter_rate'][http_request] REST_FRAMEWORK = { 'DEFAULT_RENDERER_CLASSES': ( 'rest_framework.renderers.JSONRenderer', 'swh.web.api.renderers.YAMLRenderer', 'rest_framework.renderers.TemplateHTMLRenderer' ), 'DEFAULT_THROTTLE_CLASSES': ( 'swh.web.common.throttling.SwhWebRateThrottle', ), 'DEFAULT_THROTTLE_RATES': throttle_rates } LOGGING = { 'version': 1, 'disable_existing_loggers': False, 'filters': { 'require_debug_false': { '()': 'django.utils.log.RequireDebugFalse', }, 'require_debug_true': { '()': 'django.utils.log.RequireDebugTrue', }, }, 'formatters': { 'request': { 'format': '[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, 'simple': { 'format': '[%(asctime)s] [%(levelname)s] %(message)s', 'datefmt': "%d/%b/%Y %H:%M:%S" }, 'verbose': { 'format': '[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s - %(message)s', # noqa 'datefmt': "%d/%b/%Y %H:%M:%S" }, }, 'handlers': { 'console': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', 'formatter': 'simple' }, 'file': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'simple' }, 'file_request': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'request' }, 'console_verbose': { 'level': 'DEBUG', 'filters': ['require_debug_true'], 'class': 'logging.StreamHandler', 'formatter': 'verbose' }, 'file_verbose': { 'level': 'WARNING', 'filters': ['require_debug_false'], 'class': 'logging.FileHandler', 'filename': os.path.join(swh_web_config['log_dir'], 'swh-web.log'), 'formatter': 'verbose' }, 'null': { 'class': 'logging.NullHandler', }, }, 'loggers': { '': { 'handlers': ['console_verbose', 'file_verbose'], 'level': 'DEBUG' if DEBUG else 'WARNING', }, 'django': { 'handlers': ['console'], 'level': 'DEBUG' if DEBUG else 'WARNING', 'propagate': False, }, 'django.request': { 'handlers': ['file_request'], 'level': 'DEBUG' if DEBUG else 'WARNING', 'propagate': False, }, 'django.db.backends': { 'handlers': ['null'], 'propagate': False }, }, } WEBPACK_LOADER = { # noqa 'DEFAULT': { 'CACHE': False, 'BUNDLE_DIR_NAME': './', 'STATS_FILE': os.path.join(PROJECT_DIR, '../static/webpack-stats.json'), # noqa 'POLL_INTERVAL': 0.1, 'TIMEOUT': None, 'IGNORE': ['.+\.hot-update.js', '.+\.map'] } } LOGIN_URL = '/admin/login/' LOGIN_REDIRECT_URL = 'admin' SESSION_ENGINE = 'django.contrib.sessions.backends.cache' CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache' }, 'db_cache': { 'BACKEND': 'django.core.cache.backends.db.DatabaseCache', 'LOCATION': 'swh_web_cache', } } JS_REVERSE_JS_MINIFY = False diff --git a/swh/web/wsgi.py b/swh/web/wsgi.py deleted file mode 100644 index ffd4f9be..00000000 --- a/swh/web/wsgi.py +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright (C) 2017-2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU Affero General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "swh.web.settings.production") - -application = get_wsgi_application()