diff --git a/Makefile.local b/Makefile.local index 7c9e48ea..1c04ce68 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,97 +1,94 @@ TEST_DIRS := ./swh/web/tests TESTFLAGS = --hypothesis-profile=swh-web-fast TESTFULL_FLAGS = --hypothesis-profile=swh-web YARN ?= yarn yarn-install: package.json $(YARN) install .PHONY: build-webpack-dev build-webpack-dev: yarn-install $(YARN) build-dev .PHONY: build-webpack-test build-webpack-test: yarn-install $(YARN) build-test .PHONY: build-webpack-dev-no-verbose build-webpack-dev-no-verbose: yarn-install $(YARN) build-dev >/dev/null .PHONY: build-webpack-prod build-webpack-prod: yarn-install $(YARN) build .PHONY: run-migrations run-migrations: python3 swh/web/manage.py migrate --settings=swh.web.settings.development -v0 2>/dev/null - python3 swh/web/manage.py createcachetable --settings=swh.web.settings.development -v0 2>/dev/null .PHONY: run-migrations-prod run-migrations-prod: django-admin migrate --settings=swh.web.settings.production -v0 2>/dev/null - django-admin createcachetable --settings=swh.web.settings.production -v0 2>/dev/null .PHONY: run-migrations-test run-migrations-test: rm -f swh/web/settings/testdb.sqlite3 django-admin migrate --settings=swh.web.settings.tests -v0 2>/dev/null - django-admin createcachetable --settings=swh.web.settings.tests -v0 2>/dev/null cat swh/web/tests/create_test_admin.py | django-admin shell --settings=swh.web.settings.tests .PHONY: clear-memcached clear-memcached: echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null run-django-webpack-devserver: run-migrations yarn-install bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; $(YARN) start-dev & sleep 10 && cd swh/web && python3 manage.py runserver --nostatic --settings=swh.web.settings.development" run-django-webpack-dev: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.development run-django-webpack-prod: build-webpack-prod run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-django-server-dev: run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.development run-django-server-prod: run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-gunicorn-server: run-migrations clear-memcached DJANGO_SETTINGS_MODULE=swh.web.settings.production \ gunicorn3 -b 127.0.0.1:5004 'django.core.wsgi:get_wsgi_application()' run-django-webpack-memory-storages: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests test-full: $(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS) .PHONY: test-frontend-cmd test-frontend-cmd: build-webpack-test run-migrations-test python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress run --config numTestsKeptInMemory=0 ; pkill -P $$! ; $(YARN) run mochawesome test-frontend: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend: test-frontend-cmd test-frontend-full: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full: test-frontend-cmd .PHONY: test-frontend-ui-cmd test-frontend-ui-cmd: build-webpack-test run-migrations-test bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT; jobs -p | head -1 | xargs pkill -P' SIGINT SIGTERM ERR EXIT; python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress open" test-frontend-ui: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend-ui: test-frontend-ui-cmd test-frontend-full-ui: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full-ui: test-frontend-ui-cmd # Override default rule to make sure DJANGO env var is properly set. It # *should* work without any override thanks to the mypy django-stubs plugin, # but it currently doesn't; see # https://github.com/typeddjango/django-stubs/issues/166 check-mypy: DJANGO_SETTINGS_MODULE=swh.web.settings.development $(MYPY) $(MYPYFLAGS) swh diff --git a/swh/web/misc/coverage.py b/swh/web/misc/coverage.py index 4558e38d..0f7e77ec 100644 --- a/swh/web/misc/coverage.py +++ b/swh/web/misc/coverage.py @@ -1,212 +1,161 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information from django.conf.urls import url -from django.core.cache import caches -from django.http import JsonResponse from django.shortcuts import render -from django.views.decorators.cache import never_cache from django.views.decorators.clickjacking import xframe_options_exempt -from swh.web.common import service -from swh.web.common.exc import handle_view_exception from swh.web.config import get_config # Current coverage list of the archive # TODO: Retrieve that list dynamically instead of hardcoding it _code_providers = [ { "provider_id": "bitbucket", "provider_url": "https://bitbucket.org/", "provider_logo": "img/logos/bitbucket.png", "provider_info": "public repositories from Bitbucket " "(continuously archived)", "origin_url_regexp": "^https://bitbucket.org/", "origin_types": "repositories", }, { "provider_id": "cran", "provider_url": "https://cran.r-project.org/", "provider_logo": "img/logos/cran.svg", "provider_info": "source packages from The Comprehensive R Archive " "Network (continuously archived)", "origin_url_regexp": "^https://cran.r-project.org/", "origin_types": "packages", }, { "provider_id": "debian", "provider_url": "https://www.debian.org/", "provider_logo": "img/logos/debian.png", "provider_info": "source packages from the Debian distribution " "(continuously archived)", "origin_url_regexp": "^deb://", "origin_types": "packages", }, { "provider_id": "framagit", "provider_url": "https://framagit.org/", "provider_logo": "img/logos/framagit.png", "provider_info": "public repositories from Framagit " "(continuously archived)", "origin_url_regexp": "^https://framagit.org/", "origin_types": "repositories", }, { "provider_id": "github", "provider_url": "https://github.com", "provider_logo": "img/logos/github.png", "provider_info": "public repositories from GitHub " "(continuously archived)", "origin_url_regexp": "^https://github.com/", "origin_types": "repositories", }, { "provider_id": "gitlab", "provider_url": "https://gitlab.com", "provider_logo": "img/logos/gitlab.svg", "provider_info": "public repositories from GitLab " "(continuously archived)", "origin_url_regexp": "^https://gitlab.com/", "origin_types": "repositories", }, { "provider_id": "gitorious", "provider_url": "https://gitorious.org/", "provider_logo": "img/logos/gitorious.png", "provider_info": "public repositories from the former Gitorious code " "hosting service", "origin_url_regexp": "^https://gitorious.org/", "origin_types": "repositories", }, { "provider_id": "googlecode", "provider_url": "https://code.google.com/archive/", "provider_logo": "img/logos/googlecode.png", "provider_info": "public repositories from the former Google Code " "project hosting service", "origin_url_regexp": "^http.*.googlecode.com/", "origin_types": "repositories", }, { "provider_id": "gnu", "provider_url": "https://www.gnu.org", "provider_logo": "img/logos/gnu.png", "provider_info": "releases from the GNU project (as of August 2015)", "origin_url_regexp": "^rsync://ftp.gnu.org/", "origin_types": "releases", }, { "provider_id": "hal", "provider_url": "https://hal.archives-ouvertes.fr/", "provider_logo": "img/logos/hal.png", "provider_info": "scientific software source code deposited in the " "open archive HAL", "origin_url_regexp": "^https://hal.archives-ouvertes.fr/", "origin_types": "deposits", }, { "provider_id": "inria", "provider_url": "https://gitlab.inria.fr", "provider_logo": "img/logos/inria.jpg", "provider_info": "public repositories from Inria GitLab " "(continuously archived)", "origin_url_regexp": "^https://gitlab.inria.fr/", "origin_types": "repositories", }, { "provider_id": "ipol", "provider_url": "https://www.ipol.im/", "provider_logo": "img/logos/ipol.png", "provider_info": "software artifacts associated to the articles " "IPOL publishes", "origin_url_regexp": "^https://doi.org/10.5201/ipol", "origin_types": "tarballs", }, { "provider_id": "npm", "provider_url": "https://www.npmjs.com/", "provider_logo": "img/logos/npm.png", "provider_info": "public packages from the package registry for " "javascript (continuously archived)", "origin_url_regexp": "^https://www.npmjs.com/", "origin_types": "packages", }, { "provider_id": "nixos", "provider_url": "https://nixos.org/", "provider_logo": "img/logos/nixos.png", "provider_info": "source code tarballs used to build the Nix package " "collection", "origin_url_regexp": "^https://nix-community.github.io/nixpkgs-swh", "origin_types": "tarballs", }, { "provider_id": "pypi", "provider_url": "https://pypi.org", "provider_logo": "img/logos/pypi.svg", "provider_info": "source packages from the Python Packaging Index " "(continuously archived)", "origin_url_regexp": "^https://pypi.org/", "origin_types": "packages", }, ] @xframe_options_exempt def _swh_coverage(request): count_origins = get_config()["coverage_count_origins"] return render( request, "misc/coverage.html", {"providers": _code_providers, "count_origins": count_origins}, ) -@never_cache -def _swh_coverage_count(request): - """Internal browse endpoint to count the number of origins associated - to each code provider declared in the archive coverage list. - As this operation takes some times, we execute it once per day and - cache its results to database. The cached origin counts are then served. - Cache management is handled in the implementation to avoid sending - the same count query twice to the storage database. - """ - try: - cache = caches["db_cache"] - results = [] - for code_provider in _code_providers: - provider_id = code_provider["provider_id"] - url_regexp = code_provider["origin_url_regexp"] - cache_key = "%s_origins_count" % provider_id - prev_cache_key = "%s_origins_prev_count" % provider_id - # get cached origin count - origin_count = cache.get(cache_key, -2) - # cache entry has expired or does not exist - if origin_count == -2: - # mark the origin count as processing - cache.set(cache_key, -1, timeout=10 * 60) - # execute long count query - origin_count = service.storage.origin_count(url_regexp, regexp=True) - # cache count result - cache.set(cache_key, origin_count, timeout=24 * 60 * 60) - cache.set(prev_cache_key, origin_count, timeout=None) - # origin count is currently processing - elif origin_count == -1: - # return previous count if it exists - origin_count = cache.get(prev_cache_key, -1) - results.append( - { - "provider_id": provider_id, - "origin_count": origin_count, - "origin_types": code_provider["origin_types"], - } - ) - except Exception as exc: - return handle_view_exception(request, exc, html_response=False) - - return JsonResponse(results) - - urlpatterns = [ url(r"^coverage/$", _swh_coverage, name="swh-coverage"), - url(r"^coverage/count/$", _swh_coverage_count, name="swh-coverage-count"), ] diff --git a/swh/web/settings/common.py b/swh/web/settings/common.py index 21b66414..b83a524a 100644 --- a/swh/web/settings/common.py +++ b/swh/web/settings/common.py @@ -1,284 +1,280 @@ # Copyright (C) 2017-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django common settings for swh-web. """ import os import sys from typing import Any, Dict from swh.web.config import get_config swh_web_config = get_config() # Build paths inside the project like this: os.path.join(BASE_DIR, ...) PROJECT_DIR = os.path.dirname(os.path.abspath(__file__)) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.11/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = swh_web_config["secret_key"] # SECURITY WARNING: don't run with debug turned on in production! DEBUG = swh_web_config["debug"] DEBUG_PROPAGATE_EXCEPTIONS = swh_web_config["debug"] ALLOWED_HOSTS = ["127.0.0.1", "localhost"] + swh_web_config["allowed_hosts"] # Application definition INSTALLED_APPS = [ "django.contrib.admin", "django.contrib.auth", "django.contrib.contenttypes", "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", "rest_framework", "swh.web.common", "swh.web.api", "swh.web.auth", "swh.web.browse", "webpack_loader", "django_js_reverse", "corsheaders", ] MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "swh.web.auth.middlewares.OIDCSessionRefreshMiddleware", "django.contrib.messages.middleware.MessageMiddleware", "django.middleware.clickjacking.XFrameOptionsMiddleware", "swh.web.common.middlewares.ThrottlingHeadersMiddleware", ] # Compress all assets (static ones and dynamically generated html) # served by django in a local development environment context. # In a production environment, assets compression will be directly # handled by web servers like apache or nginx. if swh_web_config["serve_assets"]: MIDDLEWARE.insert(0, "django.middleware.gzip.GZipMiddleware") ROOT_URLCONF = "swh.web.urls" TEMPLATES = [ { "BACKEND": "django.template.backends.django.DjangoTemplates", "DIRS": [os.path.join(PROJECT_DIR, "../templates")], "APP_DIRS": True, "OPTIONS": { "context_processors": [ "django.template.context_processors.debug", "django.template.context_processors.request", "django.contrib.auth.context_processors.auth", "django.contrib.messages.context_processors.messages", "swh.web.common.utils.context_processor", ], "libraries": {"swh_templatetags": "swh.web.common.swh_templatetags",}, }, }, ] DATABASES = { "default": { "ENGINE": "django.db.backends.sqlite3", "NAME": swh_web_config["development_db"], } } # Password validation # https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators AUTH_PASSWORD_VALIDATORS = [ { "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa }, {"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",}, {"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",}, {"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",}, ] # Internationalization # https://docs.djangoproject.com/en/1.11/topics/i18n/ LANGUAGE_CODE = "en-us" TIME_ZONE = "UTC" USE_I18N = True USE_L10N = True USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.11/howto/static-files/ STATIC_URL = "/static/" # static folder location when swh-web has been installed with pip STATIC_DIR = os.path.join(sys.prefix, "share/swh/web/static") if not os.path.exists(STATIC_DIR): # static folder location when developping swh-web STATIC_DIR = os.path.join(PROJECT_DIR, "../../../static") STATICFILES_DIRS = [STATIC_DIR] INTERNAL_IPS = ["127.0.0.1"] throttle_rates = {} http_requests = ["GET", "HEAD", "POST", "PUT", "DELETE", "OPTIONS", "PATCH"] throttling = swh_web_config["throttling"] for limiter_scope, limiter_conf in throttling["scopes"].items(): if "default" in limiter_conf["limiter_rate"]: throttle_rates[limiter_scope] = limiter_conf["limiter_rate"]["default"] # for backward compatibility else: throttle_rates[limiter_scope] = limiter_conf["limiter_rate"] # register sub scopes specific for HTTP request types for http_request in http_requests: if http_request in limiter_conf["limiter_rate"]: throttle_rates[limiter_scope + "_" + http_request.lower()] = limiter_conf[ "limiter_rate" ][http_request] REST_FRAMEWORK: Dict[str, Any] = { "DEFAULT_RENDERER_CLASSES": ( "rest_framework.renderers.JSONRenderer", "swh.web.api.renderers.YAMLRenderer", "rest_framework.renderers.TemplateHTMLRenderer", ), "DEFAULT_THROTTLE_CLASSES": ("swh.web.api.throttling.SwhWebRateThrottle",), "DEFAULT_THROTTLE_RATES": throttle_rates, "DEFAULT_AUTHENTICATION_CLASSES": [ "rest_framework.authentication.SessionAuthentication", "swh.web.auth.backends.OIDCBearerTokenAuthentication", ], } LOGGING = { "version": 1, "disable_existing_loggers": False, "filters": { "require_debug_false": {"()": "django.utils.log.RequireDebugFalse",}, "require_debug_true": {"()": "django.utils.log.RequireDebugTrue",}, }, "formatters": { "request": { "format": "[%(asctime)s] [%(levelname)s] %(request)s %(status_code)s", "datefmt": "%d/%b/%Y %H:%M:%S", }, "simple": { "format": "[%(asctime)s] [%(levelname)s] %(message)s", "datefmt": "%d/%b/%Y %H:%M:%S", }, "verbose": { "format": ( "[%(asctime)s] [%(levelname)s] %(name)s.%(funcName)s:%(lineno)s " "- %(message)s" ), "datefmt": "%d/%b/%Y %H:%M:%S", }, }, "handlers": { "console": { "level": "DEBUG", "filters": ["require_debug_true"], "class": "logging.StreamHandler", "formatter": "simple", }, "file": { "level": "WARNING", "filters": ["require_debug_false"], "class": "logging.FileHandler", "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), "formatter": "simple", }, "file_request": { "level": "WARNING", "filters": ["require_debug_false"], "class": "logging.FileHandler", "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), "formatter": "request", }, "console_verbose": { "level": "DEBUG", "filters": ["require_debug_true"], "class": "logging.StreamHandler", "formatter": "verbose", }, "file_verbose": { "level": "WARNING", "filters": ["require_debug_false"], "class": "logging.FileHandler", "filename": os.path.join(swh_web_config["log_dir"], "swh-web.log"), "formatter": "verbose", }, "null": {"class": "logging.NullHandler",}, }, "loggers": { "": { "handlers": ["console_verbose", "file_verbose"], "level": "DEBUG" if DEBUG else "WARNING", }, "django": { "handlers": ["console"], "level": "DEBUG" if DEBUG else "WARNING", "propagate": False, }, "django.request": { "handlers": ["file_request"], "level": "DEBUG" if DEBUG else "WARNING", "propagate": False, }, "django.db.backends": {"handlers": ["null"], "propagate": False}, "django.utils.autoreload": {"level": "INFO",}, }, } WEBPACK_LOADER = { "DEFAULT": { "CACHE": False, "BUNDLE_DIR_NAME": "./", "STATS_FILE": os.path.join(STATIC_DIR, "webpack-stats.json"), "POLL_INTERVAL": 0.1, "TIMEOUT": None, "IGNORE": [".+\\.hot-update.js", ".+\\.map"], } } LOGIN_URL = "/admin/login/" LOGIN_REDIRECT_URL = "admin" SESSION_ENGINE = "django.contrib.sessions.backends.cache" CACHES = { "default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}, - "db_cache": { - "BACKEND": "django.core.cache.backends.db.DatabaseCache", - "LOCATION": "swh_web_cache", - }, } JS_REVERSE_JS_MINIFY = False CORS_ORIGIN_ALLOW_ALL = True CORS_URLS_REGEX = r"^/badge/.*$" AUTHENTICATION_BACKENDS = [ "django.contrib.auth.backends.ModelBackend", "swh.web.auth.backends.OIDCAuthorizationCodePKCEBackend", ]