diff --git a/Makefile.local b/Makefile.local index 988d89e5..a523a65d 100644 --- a/Makefile.local +++ b/Makefile.local @@ -1,127 +1,127 @@ TEST_DIRS := ./swh/web/tests TESTFLAGS = --hypothesis-profile=swh-web-fast TESTFULL_FLAGS = --hypothesis-profile=swh-web YARN ?= yarn SETTINGS_TEST ?= swh.web.settings.tests SETTINGS_DEV ?= swh.web.settings.development SETTINGS_PROD = swh.web.settings.production yarn-install: package.json $(YARN) install --frozen-lockfile .PHONY: build-webpack-dev build-webpack-dev: yarn-install $(YARN) build-dev .PHONY: build-webpack-test build-webpack-test: yarn-install $(YARN) build-test .PHONY: build-webpack-dev-no-verbose build-webpack-dev-no-verbose: yarn-install $(YARN) build-dev >/dev/null .PHONY: build-webpack-prod build-webpack-prod: yarn-install $(YARN) build .PHONY: run-migrations-dev run-migrations-dev: python3 swh/web/manage.py rename_app --settings=$(SETTINGS_DEV) swh_web_common swh_web_save_code_now python3 swh/web/manage.py migrate --settings=$(SETTINGS_DEV) -v0 .PHONY: run-migrations-prod run-migrations-prod: django-admin rename_app --settings=$(SETTINGS_PROD) swh_web_common swh_web_save_code_now django-admin migrate --settings=$(SETTINGS_PROD) -v0 .PHONY: run-migrations-test run-migrations-test: - rm -f swh-web-test.sqlite3 + rm -f swh-web-test*.sqlite3* django-admin migrate --settings=$(SETTINGS_TEST) -v0 add-users-test: run-migrations-test cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_TEST) cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_TEST) add-users-dev: run-migrations-dev cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_DEV) cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_DEV) add-users-prod: run-migrations-prod cat swh/web/tests/create_test_admin.py | django-admin shell --settings=$(SETTINGS_PROD) cat swh/web/tests/create_test_users.py | django-admin shell --settings=$(SETTINGS_PROD) .PHONY: clear-memcached clear-memcached: echo "flush_all" | nc -q 2 localhost 11211 2>/dev/null run-django-webpack-devserver: add-users-dev yarn-install bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \ # ensure all child processes will be killed by PGID when exiting \ ps -o pgid= $$$$ | grep -o [0-9]* | xargs pkill -g' SIGINT SIGTERM ERR EXIT; \ $(YARN) start-dev & sleep 10 && cd swh/web && \ python3 manage.py runserver --nostatic --settings=$(SETTINGS_DEV) || exit 1" run-django-webpack-dev: build-webpack-dev add-users-dev python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_DEV) run-django-webpack-prod: build-webpack-prod add-users-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_PROD) run-django-server-dev: add-users-dev python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_DEV) run-django-server-prod: add-users-prod clear-memcached python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_PROD) run-gunicorn-server: add-users-prod clear-memcached DJANGO_SETTINGS_MODULE=$(SETTINGS_PROD) \ gunicorn --bind 127.0.0.1:5004 \ --threads 2 \ --workers 2 'django.core.wsgi:get_wsgi_application()' run-django-webpack-memory-storages: build-webpack-dev add-users-test python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) test-full: $(TEST) $(TESTFULL_FLAGS) $(TEST_DIRS) .PHONY: test-frontend-cmd test-frontend-cmd: build-webpack-test add-users-test bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \ jobs -p | xargs -r kill' SIGINT SIGTERM ERR EXIT; \ python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \ sleep 10 && $(YARN) run cypress run --config numTestsKeptInMemory=0 && \ $(YARN) mochawesome && $(YARN) nyc-report" test-frontend: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend: test-frontend-cmd test-frontend-full: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full: test-frontend-cmd .PHONY: test-frontend-ui-cmd test-frontend-ui-cmd: add-users-test yarn-install # ensure all child processes will be killed when hitting Ctrl-C in terminal # or manually closing the Cypress UI window, killing by PGID seems the only # reliable way to do it in that case bash -c "trap 'trap - SIGINT SIGTERM ERR EXIT && \ ps -o pgid= $$$$ | grep -o [0-9]* | xargs pkill -g' SIGINT SIGTERM ERR EXIT; \ $(YARN) start-dev & \ python3 swh/web/manage.py runserver --nostatic --settings=$(SETTINGS_TEST) & \ sleep 10 && $(YARN) run cypress open" test-frontend-ui: export CYPRESS_SKIP_SLOW_TESTS=1 test-frontend-ui: test-frontend-ui-cmd test-frontend-full-ui: export CYPRESS_SKIP_SLOW_TESTS=0 test-frontend-full-ui: test-frontend-ui-cmd # Override default rule to make sure DJANGO env var is properly set. It # *should* work without any override thanks to the mypy django-stubs plugin, # but it currently doesn't; see # https://github.com/typeddjango/django-stubs/issues/166 check-mypy: DJANGO_SETTINGS_MODULE=$(SETTINGS_DEV) $(MYPY) $(MYPYFLAGS) swh diff --git a/cypress/plugins/index.js b/cypress/plugins/index.js index a14ba40f..27e3156d 100644 --- a/cypress/plugins/index.js +++ b/cypress/plugins/index.js @@ -1,179 +1,184 @@ /** * Copyright (C) 2019-2022 The Software Heritage developers * See the AUTHORS file at the top-level directory of this distribution * License: GNU Affero General Public License version 3, or any later version * See top-level LICENSE file for more information */ const axios = require('axios'); const {execFileSync} = require('child_process'); const fs = require('fs'); const sqlite3 = require('sqlite3').verbose(); +let buildId = process.env.CYPRESS_PARALLEL_BUILD_ID; +if (buildId === undefined) { + buildId = ''; +} + async function httpGet(url) { const response = await axios.get(url); return response.data; } async function getMetadataForOrigin(originUrl, baseUrl) { const originVisitsApiUrl = `${baseUrl}/api/1/origin/${originUrl}/visits`; const originVisits = await httpGet(originVisitsApiUrl); const lastVisit = originVisits[0]; const snapshotApiUrl = `${baseUrl}/api/1/snapshot/${lastVisit.snapshot}`; const lastOriginSnapshot = await httpGet(snapshotApiUrl); let revision = lastOriginSnapshot.branches.HEAD.target; if (lastOriginSnapshot.branches.HEAD.target_type === 'alias') { revision = lastOriginSnapshot.branches[revision].target; } const revisionApiUrl = `${baseUrl}/api/1/revision/${revision}`; const lastOriginHeadRevision = await httpGet(revisionApiUrl); return { 'directory': lastOriginHeadRevision.directory, 'revision': lastOriginHeadRevision.id, 'snapshot': lastOriginSnapshot.id }; }; function getDatabase() { - const db = new sqlite3.Database('./swh-web-test.sqlite3'); + const db = new sqlite3.Database(`./swh-web-test${buildId}.sqlite3`); // to prevent "database is locked" error when running tests db.run('PRAGMA journal_mode = WAL;'); return db; } module.exports = (on, config) => { require('@cypress/code-coverage/task')(on, config); // produce JSON files prior launching browser in order to dynamically generate tests on('before:browser:launch', function(browser, launchOptions) { return new Promise((resolve) => { const p1 = axios.get(`${config.baseUrl}/tests/data/content/code/extensions/`); const p2 = axios.get(`${config.baseUrl}/tests/data/content/code/filenames/`); Promise.all([p1, p2]) .then(function(responses) { fs.writeFileSync('cypress/fixtures/source-file-extensions.json', JSON.stringify(responses[0].data)); fs.writeFileSync('cypress/fixtures/source-file-names.json', JSON.stringify(responses[1].data)); resolve(); }); }); }); on('task', { getSwhTestsData: async() => { if (!global.swhTestsData) { const swhTestsData = {}; swhTestsData.unarchivedRepo = { url: 'https://github.com/SoftwareHeritage/swh-web', type: 'git', revision: '7bf1b2f489f16253527807baead7957ca9e8adde', snapshot: 'd9829223095de4bb529790de8ba4e4813e38672d', rootDirectory: '7d887d96c0047a77e2e8c4ee9bb1528463677663', content: [{ sha1git: 'b203ec39300e5b7e97b6e20986183cbd0b797859' }] }; swhTestsData.origin = [{ url: 'https://github.com/memononen/libtess2', type: 'git', content: [{ path: 'Source/tess.h' }, { path: 'premake4.lua' }], directory: [{ path: 'Source', id: 'cd19126d815470b28919d64b2a8e6a3e37f900dd' }], revisions: [], invalidSubDir: 'Source1' }, { url: 'https://github.com/wcoder/highlightjs-line-numbers.js', type: 'git', content: [{ path: 'src/highlightjs-line-numbers.js' }], directory: [], revisions: ['1c480a4573d2a003fc2630c21c2b25829de49972'], release: { name: 'v2.6.0', id: '6877028d6e5412780517d0bfa81f07f6c51abb41', directory: '5b61d50ef35ca9a4618a3572bde947b8cccf71ad' } }]; for (const origin of swhTestsData.origin) { const metadata = await getMetadataForOrigin(origin.url, config.baseUrl); const directoryApiUrl = `${config.baseUrl}/api/1/directory/${metadata.directory}`; origin.dirContent = await httpGet(directoryApiUrl); origin.rootDirectory = metadata.directory; origin.revisions.push(metadata.revision); origin.snapshot = metadata.snapshot; for (const content of origin.content) { const contentPathApiUrl = `${config.baseUrl}/api/1/directory/${origin.rootDirectory}/${content.path}`; const contentMetaData = await httpGet(contentPathApiUrl); content.name = contentMetaData.name.split('/').slice(-1)[0]; content.sha1git = contentMetaData.target; content.directory = contentMetaData.dir_id; const rawFileUrl = `${config.baseUrl}/browse/content/sha1_git:${content.sha1git}/raw/?filename=${content.name}`; const fileText = await httpGet(rawFileUrl); const fileLines = fileText.split('\n'); content.numberLines = fileLines.length; if (!fileLines[content.numberLines - 1]) { // If last line is empty its not shown content.numberLines -= 1; } } } global.swhTestsData = swhTestsData; } return global.swhTestsData; }, 'db:user_mailmap:delete': () => { const db = getDatabase(); db.serialize(function() { db.run('DELETE FROM user_mailmap'); db.run('DELETE FROM user_mailmap_event'); }); db.close(); return true; }, 'db:user_mailmap:mark_processed': () => { const db = getDatabase(); db.serialize(function() { db.run('UPDATE user_mailmap SET mailmap_last_processing_date=datetime("now", "+1 hour")'); }); db.close(); return true; }, 'db:add_forge_now:delete': () => { const db = getDatabase(); db.serialize(function() { db.run('DELETE FROM add_forge_request_history'); db.run('DELETE FROM sqlite_sequence WHERE name="add_forge_request_history"'); }); db.serialize(function() { db.run('DELETE FROM add_forge_request'); db.run('DELETE FROM sqlite_sequence WHERE name="add_forge_request"'); }); db.close(); return true; }, processAddForgeNowInboundEmail(emailSrc) { try { execFileSync('django-admin', ['process_inbound_email', '--settings=swh.web.settings.tests'], {input: emailSrc}); return true; } catch (_) { return false; } } }); return config; }; diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py index 5160dd18..be56b29f 100644 --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -1,144 +1,148 @@ # Copyright (C) 2017-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information """ Django tests settings for swh-web. """ import os import sys from swh.web.config import get_config scope1_limiter_rate = 3 scope1_limiter_rate_post = 1 scope2_limiter_rate = 5 scope2_limiter_rate_post = 2 scope3_limiter_rate = 1 scope3_limiter_rate_post = 1 save_origin_rate_post = 5 api_raw_object_rate = 5 swh_web_config = get_config() _pytest = "pytest" in sys.argv[0] or "PYTEST_XDIST_WORKER" in os.environ swh_web_config.update( { # enable django debug mode only when running pytest "debug": _pytest, "secret_key": "test", "history_counters_url": "", "throttling": { "cache_uri": None, "scopes": { "swh_api": { "limiter_rate": {"default": "60/min"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_search": { "limiter_rate": {"default": "100/min"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_api_origin_visit_latest": { "limiter_rate": {"default": "6000/min"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_vault_cooking": { "limiter_rate": {"default": "120/h", "GET": "60/m"}, "exempted_networks": ["127.0.0.0/8"], }, "swh_save_origin": { "limiter_rate": { "default": "120/h", "POST": "%s/h" % save_origin_rate_post, } }, "swh_raw_object": { "limiter_rate": {"default": f"{api_raw_object_rate}/h"}, }, "scope1": { "limiter_rate": { "default": "%s/min" % scope1_limiter_rate, "POST": "%s/min" % scope1_limiter_rate_post, } }, "scope2": { "limiter_rate": { "default": "%s/min" % scope2_limiter_rate, "POST": "%s/min" % scope2_limiter_rate_post, } }, "scope3": { "limiter_rate": { "default": "%s/min" % scope3_limiter_rate, "POST": "%s/min" % scope3_limiter_rate_post, }, "exempted_networks": ["127.0.0.0/8"], }, }, }, "keycloak": { # disable keycloak use when not running pytest "server_url": "http://localhost:8080/auth/" if _pytest else "", "realm_name": "SoftwareHeritage", }, } ) from .common import * # noqa from .common import LOGGING # noqa, isort: skip ALLOWED_HOSTS = ["*"] DATABASES = { "default": { "ENGINE": "django.db.backends.postgresql", "NAME": swh_web_config["test_db"]["name"], } } # when running cypress tests, make the webapp fetch data from memory storages if not _pytest: swh_web_config.update( { "debug": True, "e2e_tests_mode": True, # ensure scheduler not available to avoid side effects in cypress tests "scheduler": {"cls": "remote", "url": ""}, } ) from django.conf import settings from swh.web.tests.data import get_tests_data, override_storages test_data = get_tests_data() override_storages( test_data["storage"], test_data["idx_storage"], test_data["search"], test_data["counters"], ) # using sqlite3 for frontend tests + build_id = os.environ.get("CYPRESS_PARALLEL_BUILD_ID", "") settings.DATABASES["default"].update( - {"ENGINE": "django.db.backends.sqlite3", "NAME": "swh-web-test.sqlite3"} + { + "ENGINE": "django.db.backends.sqlite3", + "NAME": f"swh-web-test{build_id}.sqlite3", + } ) # to prevent "database is locked" error when running cypress tests from django.db.backends.signals import connection_created def activate_wal_journal_mode(sender, connection, **kwargs): cursor = connection.cursor() cursor.execute("PRAGMA journal_mode = WAL;") connection_created.connect(activate_wal_journal_mode) else: # Silent DEBUG output when running unit tests LOGGING["handlers"]["console"]["level"] = "INFO" # type: ignore