diff --git a/MANIFEST.in b/MANIFEST.in --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,8 +1,8 @@ include Makefile include Makefile.local -include README.db_testing -include README.dev +include README.md include requirements.txt include requirements-swh.txt include version.txt recursive-include sql * +recursive-include swh/storage/sql * diff --git a/sql/Makefile b/sql/Makefile --- a/sql/Makefile +++ b/sql/Makefile @@ -3,30 +3,43 @@ DBNAME = softwareheritage-dev DOCDIR = autodoc -SQL_INIT = swh-init.sql -SQL_ENUMS = swh-enums.sql -SQL_SCHEMA = swh-schema.sql -SQL_FUNC = swh-func.sql -SQL_INDEX = swh-indexes.sql -SQL_TRIGGER = swh-triggers.sql +SQL_INIT = 10-swh-init.sql +SQL_ENUMS = 20-swh-enums.sql +SQL_SCHEMA = 30-swh-schema.sql +SQL_FUNC = 40-swh-func.sql +SQL_INDEX = 60-swh-indexes.sql +SQL_TRIGGER = 70-swh-triggers.sql SQLS = $(SQL_INIT) $(SQL_ENUMS) $(SQL_SCHEMA) $(SQL_FUNC) $(SQL_INDEX) $(SQL_TRIGGER) +SQL_FILES = $(abspath $(addprefix $(CURDIR)/../swh/storage/sql/,$(SQLS))) PSQL_BIN = psql PSQL_FLAGS = --echo-all -X -v ON_ERROR_STOP=1 PSQL = $(PSQL_BIN) $(PSQL_FLAGS) +PIFPAF=$(findstring postgresql://,$(PIFPAF_URLS)) + all: createdb: createdb-stamp -createdb-stamp: $(SQL_INIT) +createdb-stamp: $(SQL_FILES) +ifndef PIFPAF -dropdb $(DBNAME) +endif createdb $(DBNAME) +ifndef PIFPAF touch $@ +else + rm -f $@ +endif filldb: filldb-stamp filldb-stamp: createdb-stamp - cat $(SQLS) | $(PSQL) $(DBNAME) + cat $(SQL_FILES) | $(PSQL) $(DBNAME) +ifndef PIFPAF touch $@ +else + rm -f $@ +endif dropdb: -dropdb $(DBNAME) @@ -42,7 +55,11 @@ autodoc-stamp: filldb-stamp $(DOCDIR) postgresql_autodoc -d $(DBNAME) -f $(DOCDIR)/db-schema cp -a $(DOCDIR)/db-schema.dot $(DOCDIR)/db-schema.dot.orig +ifndef PIFPAF touch $@ +else + rm -f $@ +endif $(DOCDIR)/db-schema.dot: clusters.dot autodoc-stamp $(DOCDIR) bin/dot_add_content $(DOCDIR)/db-schema.dot.orig clusters.dot > $(DOCDIR)/db-schema.dot diff --git a/sql/bin/db-init b/sql/bin/db-init deleted file mode 100755 --- a/sql/bin/db-init +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash -set -e - -# Must be run as (a) Postgres super user - -SQL_INIT=swh-init.sql -SQL_ENUMS=swh-enums.sql -SQL_SCHEMA=swh-schema.sql -SQL_FUNC=swh-func.sql -SQL_INDEX=swh-indexes.sql -SQL_TRIGGER=swh-triggers.sql -SQL_DATA=swh-data.sql -ROOT_SQLS="$SQL_INIT" -USER_SQLS="$SQL_ENUMS $SQL_SCHEMA $SQL_FUNC $SQL_INDEX $SQL_TRIGGER $SQL_DATA" -SQLS="$ROOT_SQLS $USER_SQLS" - -DB_ENCODING="UTF-8" -DB_LOCALE="C.UTF-8" -DB_TEMPLATE="template0" - -cd "$( dirname $0 )/.." - -if ! [ -f "$SQL_INIT" ] ; then - echo "Cannot find $SQL_INIT. Abort." - exit 2 -fi - -if [ -z "$1" ] ; then - echo "Usage: bin/db-init DB_NAME [DB_PORT]" - echo "Example: bin/db-init softwareheritage-dev" - echo "Note: DB_NAME should not exist and will be created" - exit 2 -fi -db_name="$1" -port=${2:-5432} - -conn_flags="--port $port" - -echo "I: creating Postgres database ${db_name} ..." -createdb $conn_flags \ - --encoding "$DB_ENCODING" --locale "$DB_LOCALE" \ - --template "$DB_TEMPLATE" "$db_name" - -sqls_flags='' -for f in $SQLS ; do - sqls_flags="${sqls_flags} --file ${f}" -done - -echo "I: initializing DB ${db_name} ..." -psql $conn_flags ${sqls_flags} "$db_name" - -echo "I: all done." diff --git a/sql/swh-init.sql b/swh/storage/sql/10-swh-init.sql rename from sql/swh-init.sql rename to swh/storage/sql/10-swh-init.sql diff --git a/sql/swh-enums.sql b/swh/storage/sql/20-swh-enums.sql rename from sql/swh-enums.sql rename to swh/storage/sql/20-swh-enums.sql diff --git a/sql/swh-schema.sql b/swh/storage/sql/30-swh-schema.sql rename from sql/swh-schema.sql rename to swh/storage/sql/30-swh-schema.sql diff --git a/sql/swh-func.sql b/swh/storage/sql/40-swh-func.sql rename from sql/swh-func.sql rename to swh/storage/sql/40-swh-func.sql diff --git a/sql/swh-indexes.sql b/swh/storage/sql/60-swh-indexes.sql rename from sql/swh-indexes.sql rename to swh/storage/sql/60-swh-indexes.sql diff --git a/sql/swh-triggers.sql b/swh/storage/sql/70-swh-triggers.sql rename from sql/swh-triggers.sql rename to swh/storage/sql/70-swh-triggers.sql diff --git a/swh/storage/tests/__init__.py b/swh/storage/tests/__init__.py --- a/swh/storage/tests/__init__.py +++ b/swh/storage/tests/__init__.py @@ -0,0 +1,5 @@ +from os import path +import swh.storage + + +SQL_DIR = path.join(path.dirname(swh.storage.__file__), 'sql') diff --git a/swh/storage/tests/storage_testing.py b/swh/storage/tests/storage_testing.py --- a/swh/storage/tests/storage_testing.py +++ b/swh/storage/tests/storage_testing.py @@ -3,36 +3,30 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os import tempfile -import pathlib from swh.storage import get_storage +from swh.core.tests.db_testing import SingleDbTestFixture +from swh.storage.tests import SQL_DIR -class StorageTestFixture: + +class StorageTestFixture(SingleDbTestFixture): """Mix this in a test subject class to get Storage testing support. - This fixture requires to come before DbTestFixture in the inheritance list - as it uses its methods to setup its own internal database. + This fixture requires to come before SingleDbTestFixture in the + inheritance list as it uses its methods to setup its own + internal database. Usage example: - class TestStorage(StorageTestFixture, DbTestFixture): + class MyTestStorage(StorageTestFixture, unittest.TestCase): ... - """ - TEST_STORAGE_DB_NAME = 'softwareheritage-test-storage' - @classmethod - def setUpClass(cls): - if not hasattr(cls, 'DB_TEST_FIXTURE_IMPORTED'): - raise RuntimeError("StorageTestFixture needs to be followed by " - "DbTestFixture in the inheritance list.") - - test_dir = pathlib.Path(__file__).absolute().parent - test_data_dir = test_dir / '../../../../swh-storage-testdata' - test_db_dump = (test_data_dir / 'dumps/swh.dump').absolute() - cls.add_db(cls.TEST_STORAGE_DB_NAME, str(test_db_dump), 'pg_dump') - super().setUpClass() + """ + TEST_DB_NAME = 'softwareheritage-test-storage' + TEST_DB_DUMP = os.path.join(SQL_DIR, '*.sql') def setUp(self): super().setUp() @@ -41,7 +35,7 @@ self.storage_config = { 'cls': 'local', 'args': { - 'db': 'dbname=%s' % self.TEST_STORAGE_DB_NAME, + 'db': 'dbname=%s' % self.TEST_DB_NAME, 'objstorage': { 'cls': 'pathslicing', 'args': { @@ -60,4 +54,4 @@ def reset_storage_tables(self): excluded = {'dbversion', 'tool'} - self.reset_db_tables(self.TEST_STORAGE_DB_NAME, excluded=excluded) + self.reset_db_tables(self.TEST_DB_NAME, excluded=excluded) diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -33,7 +33,7 @@ 'storage': { 'cls': 'local', 'args': { - 'db': 'dbname=%s' % self.TEST_STORAGE_DB_NAME, + 'db': 'dbname=%s' % self.TEST_DB_NAME, 'objstorage': { 'cls': 'pathslicing', 'args': { diff --git a/swh/storage/tests/test_db.py b/swh/storage/tests/test_db.py --- a/swh/storage/tests/test_db.py +++ b/swh/storage/tests/test_db.py @@ -11,15 +11,13 @@ from swh.core.tests.db_testing import SingleDbTestFixture from swh.model.hashutil import hash_to_bytes from swh.storage.db import Db - -TEST_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA_DIR = os.path.join(TEST_DIR, '../../../../swh-storage-testdata') +from . import SQL_DIR @attr('db') class TestDb(SingleDbTestFixture, unittest.TestCase): - - TEST_DB_DUMP = os.path.join(TEST_DATA_DIR, 'dumps/swh.dump') + TEST_DB_NAME = 'softwareheritage-test-storage' + TEST_DB_DUMP = os.path.join(SQL_DIR, '*.sql') def setUp(self): super().setUp() diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -20,11 +20,11 @@ @attr('db') -class BaseTestStorage(StorageTestFixture, DbTestFixture): +class BaseTestStorage(StorageTestFixture): def setUp(self): super().setUp() - db = self.test_db[self.TEST_STORAGE_DB_NAME] + db = self.test_db[self.TEST_DB_NAME] self.conn = db.conn self.cursor = db.cursor