Page MenuHomeSoftware Heritage
Files F9696949

No OneTemporary
Actions

View Options

diff --git a/requirements.txt b/requirements.txt
index 887e4561..fb4c3b49 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,24 +1,25 @@
# Add here external Python modules dependencies, one per line. Module names
# should match https://pypi.python.org/pypi names. For the full spec or
# dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html
beautifulsoup4
cryptography
django < 3
django-cors-headers
django-js-reverse
djangorestframework
django-webpack-loader
docutils
htmlmin
iso8601
lxml
prometheus-client
pybadges
pygments
python-magic >= 0.4.0
python-memcached
pyyaml
requests
sentry-sdk
typing-extensions
+psycopg2
diff --git a/swh/web/config.py b/swh/web/config.py
index e6ddfb88..c9d3f1d2 100644
--- a/swh/web/config.py
+++ b/swh/web/config.py
@@ -1,199 +1,199 @@
# Copyright (C) 2017-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
from typing import Any, Dict
from swh.core import config
from swh.indexer.storage import get_indexer_storage
from swh.scheduler import get_scheduler
from swh.search import get_search
from swh.storage import get_storage
from swh.vault import get_vault
from swh.web import settings
SWH_WEB_INTERNAL_SERVER_NAME = "archive.internal.softwareheritage.org"
STAGING_SERVER_NAMES = [
"webapp.staging.swh.network",
"webapp.internal.staging.swh.network",
]
ORIGIN_VISIT_TYPES = [
"cran",
"deb",
"deposit",
"ftp",
"hg",
"git",
"nixguix",
"npm",
"pypi",
"svn",
"tar",
]
SETTINGS_DIR = os.path.dirname(settings.__file__)
DEFAULT_CONFIG = {
"allowed_hosts": ("list", []),
"search": (
"dict",
{"cls": "remote", "url": "http://127.0.0.1:5010/", "timeout": 10,},
),
"storage": (
"dict",
{"cls": "remote", "url": "http://127.0.0.1:5002/", "timeout": 10,},
),
"indexer_storage": (
"dict",
{"cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1,},
),
"log_dir": ("string", "/tmp/swh/log"),
"debug": ("bool", False),
"serve_assets": ("bool", False),
"host": ("string", "127.0.0.1"),
"port": ("int", 5004),
"secret_key": ("string", "development key"),
# do not display code highlighting for content > 1MB
"content_display_max_size": ("int", 5 * 1024 * 1024),
"snapshot_content_max_size": ("int", 1000),
"throttling": (
"dict",
{
"cache_uri": None, # production: memcached as cache (127.0.0.1:11211)
# development: in-memory cache so None
"scopes": {
"swh_api": {
"limiter_rate": {"default": "120/h"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_api_origin_search": {
"limiter_rate": {"default": "10/m"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_vault_cooking": {
"limiter_rate": {"default": "120/h", "GET": "60/m"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_save_origin": {
"limiter_rate": {"default": "120/h", "POST": "10/h"},
"exempted_networks": ["127.0.0.0/8"],
},
"swh_api_origin_visit_latest": {
"limiter_rate": {"default": "700/m"},
"exempted_networks": ["127.0.0.0/8"],
},
},
},
),
"vault": ("dict", {"cls": "remote", "args": {"url": "http://127.0.0.1:5005/",}}),
"scheduler": ("dict", {"cls": "remote", "url": "http://127.0.0.1:5008/"}),
"development_db": ("string", os.path.join(SETTINGS_DIR, "db.sqlite3")),
"test_db": ("string", os.path.join(SETTINGS_DIR, "testdb.sqlite3")),
- "production_db": ("string", "/var/lib/swh/web.sqlite3"),
+ "production_db": ("dict", {"name": "swh-web"}),
"deposit": (
"dict",
{
"private_api_url": "https://deposit.softwareheritage.org/1/private/",
"private_api_user": "swhworker",
"private_api_password": "",
},
),
"coverage_count_origins": ("bool", False),
"e2e_tests_mode": ("bool", False),
"es_workers_index_url": ("string", ""),
"history_counters_url": (
"string",
"https://stats.export.softwareheritage.org/history_counters.json",
),
"client_config": ("dict", {}),
"keycloak": ("dict", {"server_url": "", "realm_name": ""}),
"graph": (
"dict",
{"server_url": "http://graph.internal.softwareheritage.org:5009/graph/"},
),
"status": (
"dict",
{
"server_url": "https://status.softwareheritage.org/",
"json_path": "1.0/status/578e5eddcdc0cc7951000520",
},
),
"metadata_search_backend": ("string", "swh-indexer-storage"), # or "swh-search"
"staging_server_names": ("list", STAGING_SERVER_NAMES),
}
swhweb_config = {} # type: Dict[str, Any]
def get_config(config_file="web/web"):
"""Read the configuration file `config_file`.
If an environment variable SWH_CONFIG_FILENAME is defined, this
takes precedence over the config_file parameter.
In any case, update the app with parameters (secret_key, conf)
and return the parsed configuration as a dict.
If no configuration file is provided, return a default
configuration.
"""
if not swhweb_config:
config_filename = os.environ.get("SWH_CONFIG_FILENAME")
if config_filename:
config_file = config_filename
cfg = config.load_named_config(config_file, DEFAULT_CONFIG)
swhweb_config.update(cfg)
config.prepare_folders(swhweb_config, "log_dir")
if swhweb_config.get("search"):
swhweb_config["search"] = get_search(**swhweb_config["search"])
else:
swhweb_config["search"] = None
swhweb_config["storage"] = get_storage(**swhweb_config["storage"])
swhweb_config["vault"] = get_vault(**swhweb_config["vault"])
swhweb_config["indexer_storage"] = get_indexer_storage(
**swhweb_config["indexer_storage"]
)
swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"])
return swhweb_config
def search():
"""Return the current application's search.
"""
return get_config()["search"]
def storage():
"""Return the current application's storage.
"""
return get_config()["storage"]
def vault():
"""Return the current application's vault.
"""
return get_config()["vault"]
def indexer_storage():
"""Return the current application's indexer storage.
"""
return get_config()["indexer_storage"]
def scheduler():
"""Return the current application's scheduler.
"""
return get_config()["scheduler"]
diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py
index 4ed4e495..d61949b6 100644
--- a/swh/web/settings/common.py
+++ b/swh/web/settings/common.py
@@ -1,288 +1,288 @@
-# Copyright (C) 2017-2020 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
Django common settings for swh-web.
"""
import os
import sys
from typing import Any, Dict
from swh.web.auth.utils import OIDC_SWH_WEB_CLIENT_ID
from swh.web.config import get_config
swh_web_config = get_config()
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
PROJECT_DIR = os.path.dirname(os.path.abspath(__file__))
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = swh_web_config["secret_key"]
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = swh_web_config["debug"]
DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config["debug"]
ALLOWED_HOSTS = ["127.0.0.1", "localhost"] + swh_web_config["allowed_hosts"]
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"rest_framework",
"swh.web.common",
"swh.web.api",
"swh.web.auth",
"swh.web.browse",
"webpack_loader",
"django_js_reverse",
"corsheaders",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"corsheaders.middleware.CorsMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"swh.auth.django.middlewares.OIDCSessionExpiredMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
"swh.web.common.middlewares.ThrottlingHeadersMiddleware",
"swh.web.common.middlewares.ExceptionMiddleware",
]
# Compress all assets (static ones and dynamically generated html)
# served by django in a local development environment context.
# In a production environment, assets compression will be directly
# handled by web servers like apache or nginx.
if swh_web_config["serve_assets"]:
MIDDLEWARE.insert(0, "django.middleware.gzip.GZipMiddleware")
ROOT_URLCONF = "swh.web.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [os.path.join(PROJECT_DIR, "../templates")],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
"swh.web.common.utils.context_processor",
],
"libraries": {"swh_templatetags": "swh.web.common.swh_templatetags",},
},
},
]
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
- "NAME": swh_web_config["development_db"],
+ "NAME": swh_web_config.get("development_db", ""),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa
},
{"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",},
{"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",},
{"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",},
]
# Internationalization
# https://docs.djangoproject.com/en/1.11/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/1.11/howto/static-files/
STATIC_URL = "/static/"
# static folder location when swh-web has been installed with pip
STATIC_DIR = os.path.join(sys.prefix, "share/swh/web/static")
if not os.path.exists(STATIC_DIR):
# static folder location when developping swh-web
STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static")
STATICFILES_DIRS = [STATIC_DIR]
INTERNAL_IPS = ["127.0.0.1"]
throttle_rates = {}
http_requests = ["GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"]
throttling = swh_web_config["throttling"]
for limiter_scope, limiter_conf in throttling["scopes"].items():
if "default" in limiter_conf["limiter_rate"]:
throttle_rates[limiter_scope] = limiter_conf["limiter_rate"]["default"]
# for backward compatibility
else:
throttle_rates[limiter_scope] = limiter_conf["limiter_rate"]
# register sub scopes specific for HTTP request types
for http_request in http_requests:
if http_request in limiter_conf["limiter_rate"]:
throttle_rates[limiter_scope + "_" + http_request.lower()] = limiter_conf[
"limiter_rate"
][http_request]
REST_FRAMEWORK: Dict[str, Any] = {
"DEFAULT_RENDERER_CLASSES": (
"rest_framework.renderers.JSONRenderer",
"swh.web.api.renderers.YAMLRenderer",
"rest_framework.renderers.TemplateHTMLRenderer",
),
"DEFAULT_THROTTLE_CLASSES": ("swh.web.api.throttling.SwhWebRateThrottle",),
"DEFAULT_THROTTLE_RATES": throttle_rates,
"DEFAULT_AUTHENTICATION_CLASSES": [
"rest_framework.authentication.SessionAuthentication",
"swh.auth.django.backends.OIDCBearerTokenAuthentication",
],
"EXCEPTION_HANDLER": "swh.web.api.apiresponse.error_response_handler",
}
LOGGING = {
"version": 1,
"disable_existing_loggers": False,
"filters": {
"require_debug_false": {"()": "django.utils.log.RequireDebugFalse",},
"require_debug_true": {"()": "django.utils.log.RequireDebugTrue",},
},
"formatters": {
"request": {
"format": "[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s",
"datefmt": "%d/%b/%Y %H:%M:%S",
},
"simple": {
"format": "[%(asctime)s] [%(levelname)s] %(message)s",
"datefmt": "%d/%b/%Y %H:%M:%S",
},
"verbose": {
"format": (
"[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s "
"- %(message)s"
),
"datefmt": "%d/%b/%Y %H:%M:%S",
},
},
"handlers": {
"console": {
"level": "DEBUG",
"filters": ["require_debug_true"],
"class": "logging.StreamHandler",
"formatter": "simple",
},
"file": {
"level": "WARNING",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"),
"formatter": "simple",
},
"file_request": {
"level": "WARNING",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"),
"formatter": "request",
},
"console_verbose": {
"level": "DEBUG",
"filters": ["require_debug_true"],
"class": "logging.StreamHandler",
"formatter": "verbose",
},
"file_verbose": {
"level": "WARNING",
"filters": ["require_debug_false"],
"class": "logging.FileHandler",
"filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"),
"formatter": "verbose",
},
"null": {"class": "logging.NullHandler",},
},
"loggers": {
"": {
"handlers": ["console_verbose", "file_verbose"],
"level": "DEBUG" if DEBUG else "WARNING",
},
"django": {
"handlers": ["console"],
"level": "DEBUG" if DEBUG else "WARNING",
"propagate": False,
},
"django.request": {
"handlers": ["file_request"],
"level": "DEBUG" if DEBUG else "WARNING",
"propagate": False,
},
"django.db.backends": {"handlers": ["null"], "propagate": False},
"django.utils.autoreload": {"level": "INFO",},
},
}
WEBPACK_LOADER = {
"DEFAULT": {
"CACHE": False,
"BUNDLE_DIR_NAME": "./",
"STATS_FILE": os.path.join(STATIC_DIR, "webpack-stats.json"),
"POLL_INTERVAL": 0.1,
"TIMEOUT": None,
"IGNORE": [".+\\.hot-update.js", ".+\\.map"],
}
}
LOGIN_URL = "/admin/login/"
LOGIN_REDIRECT_URL = "admin"
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
CACHES = {
"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"},
}
JS_REVERSE_JS_MINIFY = False
CORS_ORIGIN_ALLOW_ALL = True
CORS_URLS_REGEX = r"^/(badge|api)/.*$"
AUTHENTICATION_BACKENDS = [
"django.contrib.auth.backends.ModelBackend",
"swh.auth.django.backends.OIDCAuthorizationCodePKCEBackend",
]
SWH_AUTH_SERVER_URL = swh_web_config["keycloak"]["server_url"]
SWH_AUTH_REALM_NAME = swh_web_config["keycloak"]["realm_name"]
SWH_AUTH_CLIENT_ID = OIDC_SWH_WEB_CLIENT_ID
SWH_AUTH_SESSION_EXPIRED_REDIRECT_VIEW = "logout"
diff --git a/swh/web/settings/production.py b/swh/web/settings/production.py
index a9894c8c..a15ead4f 100644
--- a/swh/web/settings/production.py
+++ b/swh/web/settings/production.py
@@ -1,41 +1,48 @@
-# Copyright (C) 2017-2019 The Software Heritage developers
+# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information
"""
Django production settings for swh-web.
"""
from .common import * # noqa
from .common import CACHES, MIDDLEWARE, REST_FRAMEWORK, WEBPACK_LOADER, swh_web_config
MIDDLEWARE += [
"swh.web.common.middlewares.HtmlMinifyMiddleware",
]
if swh_web_config.get("throttling", {}).get("cache_uri"):
CACHES.update(
{
"default": {
"BACKEND": "django.core.cache.backends.memcached.MemcachedCache",
"LOCATION": swh_web_config["throttling"]["cache_uri"],
}
}
)
# Setup support for proxy headers
USE_X_FORWARDED_HOST = True
SECURE_PROXY_SSL_HEADER = ("HTTP_X_FORWARDED_PROTO", "https")
# We're going through seven (or, in that case, 2) proxies thanks to Varnish
REST_FRAMEWORK["NUM_PROXIES"] = 2
+db_conf = swh_web_config["production_db"]
+
+# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
DATABASES = {
"default": {
- "ENGINE": "django.db.backends.sqlite3",
- "NAME": swh_web_config["production_db"],
+ "ENGINE": "django.db.backends.postgresql",
+ "NAME": db_conf.get("name"),
+ "HOST": db_conf.get("host"),
+ "PORT": db_conf.get("port"),
+ "USER": db_conf.get("user"),
+ "PASSWORD": db_conf.get("password"),
}
}
WEBPACK_LOADER["DEFAULT"]["CACHE"] = True

File Metadata

Mime Type
text/x-diff
Expires
Mon, Aug 18, 10:16 PM (5 d, 5 m ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3354189

Event Timeline

About · Developer information · Legal