diff --git a/Makefile.local b/Makefile.local index 6e48acc2..46177f72 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,68 +1,75 @@ TEST_DIRS := ./swh/web/tests TESTFLAGS = --hypothesis-profile=swh-web-fast TESTFULL_FLAGS = --hypothesis-profile=swh-web YARN ?= yarn yarn-install: package.json $(YARN) install .PHONY: build-webpack-dev build-webpack-dev: yarn-install $(YARN) build-dev .PHONY: build-webpack-test build-webpack-test: yarn-install $(YARN) build-test .PHONY: build-webpack-dev-no-verbose build-webpack-dev-no-verbose: yarn-install $(YARN) build-dev >/dev/null .PHONY: build-webpack-prod build-webpack-prod: yarn-install $(YARN) build .PHONY: run-migrations run-migrations: python3 swh/web/manage.py migrate -v0 2>/dev/null python3 swh/web/manage.py createcachetable -v0 2>/dev/null .PHONY: run-migrations-prod run-migrations-prod: django-admin migrate --settings=swh.web.settings.production -v0 2>/dev/null django-admin createcachetable --settings=swh.web.settings.production -v0 2>/dev/null +.PHONY: run-migrations-test +run-migrations-test: + rm -f swh/web/settings/testdb.sqlite3 + django-admin migrate --settings=swh.web.settings.tests -v0 2>/dev/null + django-admin createcachetable --settings=swh.web.settings.tests -v0 2>/dev/null + cat swh/web/tests/create_test_admin.py | django-admin shell --settings=swh.web.settings.tests + .PHONY: clear-memcached clear-memcached: echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null run-django-webpack-devserver: run-migrations yarn-install bash -c "trap 'trap - SIGINT SIGTERM ERR; kill %1' SIGINT SIGTERM ERR; $(YARN) start-dev & sleep 10 && cd swh/web && python3 manage.py runserver --nostatic" run-django-webpack-dev: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic run-django-webpack-prod: build-webpack-prod run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-django-server-dev: run-migrations python3 swh/web/manage.py runserver --nostatic run-django-server-prod: run-migrations-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.production run-gunicorn-server: run-migrations clear-memcached gunicorn3 -b 127.0.0.1:5004 swh.web.wsgi run-django-webpack-memory-storages: build-webpack-dev run-migrations python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests test-full: $(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS) -test-frontend: build-webpack-test run-migrations +test-frontend: build-webpack-test run-migrations-test python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress run && pkill -P $$! && $(YARN) run mochawesome -test-frontend-ui: build-webpack-test run-migrations +test-frontend-ui: build-webpack-test run-migrations-test bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT; jobs -p | head -1 | xargs pkill -P' SIGINT SIGTERM ERR EXIT; python3 swh/web/manage.py runserver --nostatic --settings=swh.web.settings.tests & sleep 10 && $(YARN) run cypress open" diff --git a/swh/web/config.py b/swh/web/config.py index 5cecf2f4..635d5f24 100644 --- a/swh/web/config.py +++ b/swh/web/config.py @@ -1,155 +1,156 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information import os from swh.core import config from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.storage import get_storage from swh.vault import get_vault from swh.web import settings SETTINGS_DIR = os.path.dirname(settings.__file__) DEFAULT_CONFIG = { 'allowed_hosts': ('list', []), 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5002/', 'timeout': 10, }, }), 'indexer_storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5007/', 'timeout': 1, } }), 'log_dir': ('string', '/tmp/swh/log'), 'debug': ('bool', False), 'serve_assets': ('bool', False), 'host': ('string', '127.0.0.1'), 'port': ('int', 5004), 'secret_key': ('string', 'development key'), # do not display code highlighting for content > 1MB 'content_display_max_size': ('int', 5 * 1024 * 1024), 'snapshot_content_max_size': ('int', 1000), 'throttling': ('dict', { 'cache_uri': None, # production: memcached as cache (127.0.0.1:11211) # development: in-memory cache so None 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '120/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '10/h' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_api_origin_visit_latest': { 'limiter_rate': { 'default': '700/m' }, 'exempted_networks': ['127.0.0.0/8'], }, } }), 'vault': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5005/', } }), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://127.0.0.1:5008/' } }), 'development_db': ('string', os.path.join(SETTINGS_DIR, 'db.sqlite3')), + 'test_db': ('string', os.path.join(SETTINGS_DIR, 'testdb.sqlite3')), 'production_db': ('string', '/var/lib/swh/web.sqlite3'), 'deposit': ('dict', { 'private_api_url': 'https://deposit.softwareheritage.org/1/private/', 'private_api_user': 'swhworker', 'private_api_password': '' }), 'coverage_count_origins': ('bool', False), 'e2e_tests_mode': ('bool', False) } swhweb_config = {} def get_config(config_file='web/web'): """Read the configuration file `config_file`. If an environment variable SWH_CONFIG_FILENAME is defined, this takes precedence over the config_file parameter. In any case, update the app with parameters (secret_key, conf) and return the parsed configuration as a dict. If no configuration file is provided, return a default configuration. """ if not swhweb_config: config_filename = os.environ.get('SWH_CONFIG_FILENAME') if config_filename: config_file = config_filename cfg = config.load_named_config(config_file, DEFAULT_CONFIG) swhweb_config.update(cfg) config.prepare_folders(swhweb_config, 'log_dir') swhweb_config['storage'] = get_storage(**swhweb_config['storage']) swhweb_config['vault'] = get_vault(**swhweb_config['vault']) swhweb_config['indexer_storage'] = \ get_indexer_storage(**swhweb_config['indexer_storage']) swhweb_config['scheduler'] = get_scheduler( **swhweb_config['scheduler']) return swhweb_config def storage(): """Return the current application's storage. """ return get_config()['storage'] def vault(): """Return the current application's vault. """ return get_config()['vault'] def indexer_storage(): """Return the current application's indexer storage. """ return get_config()['indexer_storage'] def scheduler(): """Return the current application's scheduler. """ return get_config()['scheduler'] diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py index a3becee4..ced666b3 100644 --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -1,95 +1,102 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django tests settings for swh-web. """ import sys from swh.web.config import get_config scope1_limiter_rate = 3 scope1_limiter_rate_post = 1 scope2_limiter_rate = 5 scope2_limiter_rate_post = 2 scope3_limiter_rate = 1 scope3_limiter_rate_post = 1 save_origin_rate_post = 10 swh_web_config = get_config() swh_web_config.update({ 'debug': False, 'secret_key': 'test', 'throttling': { 'cache_uri': None, 'scopes': { 'swh_api': { 'limiter_rate': { 'default': '60/min' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_api_origin_visit_latest': { 'limiter_rate': { 'default': '6000/min' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_vault_cooking': { 'limiter_rate': { 'default': '120/h', 'GET': '60/m' }, 'exempted_networks': ['127.0.0.0/8'] }, 'swh_save_origin': { 'limiter_rate': { 'default': '120/h', 'POST': '%s/h' % save_origin_rate_post, } }, 'scope1': { 'limiter_rate': { 'default': '%s/min' % scope1_limiter_rate, 'POST': '%s/min' % scope1_limiter_rate_post, } }, 'scope2': { 'limiter_rate': { 'default': '%s/min' % scope2_limiter_rate, 'POST': '%s/min' % scope2_limiter_rate_post } }, 'scope3': { 'limiter_rate': { 'default': '%s/min' % scope3_limiter_rate, 'POST': '%s/min' % scope3_limiter_rate_post }, 'exempted_networks': ['127.0.0.0/8'] } } } }) from .common import * # noqa from .common import ALLOWED_HOSTS, LOGGING # noqa +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': swh_web_config['test_db'], + } +} + # when not running unit tests, make the webapp fetch data from memory storages if 'pytest' not in sys.argv[0]: swh_web_config.update({ 'debug': True, 'e2e_tests_mode': True }) from swh.web.tests.data import get_tests_data, override_storages # noqa test_data = get_tests_data() override_storages(test_data['storage'], test_data['idx_storage']) else: ALLOWED_HOSTS += ['testserver'] # Silent DEBUG output when running unit tests LOGGING['handlers']['console']['level'] = 'INFO' diff --git a/swh/web/tests/create_test_admin.py b/swh/web/tests/create_test_admin.py new file mode 100644 index 00000000..1e1274c0 --- /dev/null +++ b/swh/web/tests/create_test_admin.py @@ -0,0 +1,16 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + + +from django.contrib.auth import get_user_model + +username = 'admin' +password = 'admin' +email = 'admin@swh-web.org' + +User = get_user_model() + +if not User.objects.filter(username=username).exists(): + User.objects.create_superuser(username, email, password)