diff --git a/requirements-db-pytestplugin.txt b/requirements-db-pytestplugin.txt --- a/requirements-db-pytestplugin.txt +++ b/requirements-db-pytestplugin.txt @@ -1,2 +1,2 @@ # requirements for swh.core.db.pytest_plugin -pytest-postgresql < 3.0.0 +pytest-postgresql diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -43,6 +43,7 @@ @db.command(name="create", context_settings=CONTEXT_SETTINGS) @click.argument("module", required=True) @click.option( + "--dbname", "--db-name", "-d", help="Database name.", @@ -56,7 +57,7 @@ default="template1", show_default=True, ) -def db_create(module, db_name, template): +def db_create(module, dbname, template): """Create a database for the Software Heritage . and potentially execute superuser-level initialization steps. @@ -79,20 +80,21 @@ """ - logger.debug("db_create %s dn_name=%s", module, db_name) - create_database_for_package(module, db_name, template) + logger.debug("db_create %s dn_name=%s", module, dbname) + create_database_for_package(module, dbname, template) @db.command(name="init-admin", context_settings=CONTEXT_SETTINGS) @click.argument("module", required=True) @click.option( + "--dbname", "--db-name", "-d", help="Database name.", default="softwareheritage-dev", show_default=True, ) -def db_init_admin(module: str, db_name: str) -> None: +def db_init_admin(module: str, dbname: str) -> None: """Execute superuser-level initialization steps (e.g pg extensions, admin functions, ...) @@ -115,13 +117,14 @@ scheduler """ - logger.debug("db_init_admin %s db_name=%s", module, db_name) - init_admin_extensions(module, db_name) + logger.debug("db_init_admin %s dbname=%s", module, dbname) + init_admin_extensions(module, dbname) @db.command(name="init", context_settings=CONTEXT_SETTINGS) @click.argument("module", required=True) @click.option( + "--dbname", "--db-name", "-d", help="Database name.", @@ -131,7 +134,7 @@ @click.option( "--flavor", help="Database flavor.", default=None, ) -def db_init(module, db_name, flavor): +def db_init(module, dbname, flavor): """Initialize a database for the Software Heritage . Example:: @@ -150,10 +153,10 @@ """ - logger.debug("db_init %s flavor=%s dn_name=%s", module, flavor, db_name) + logger.debug("db_init %s flavor=%s dbname=%s", module, flavor, dbname) initialized, dbversion, dbflavor = populate_database_for_package( - module, db_name, flavor + module, dbname, flavor ) # TODO: Ideally migrate the version from db_version to the latest @@ -274,9 +277,9 @@ # Use the given conninfo string, but with dbname replaced by the template dbname # for the database creation step creation_dsn = parse_dsn_or_dbname(conninfo) - db_name = creation_dsn["dbname"] + dbname = creation_dsn["dbname"] creation_dsn["dbname"] = template - logger.debug("db_create db_name=%s (from %s)", db_name, template) + logger.debug("db_create dbname=%s (from %s)", dbname, template) subprocess.check_call( [ "psql", @@ -287,7 +290,7 @@ "-d", make_dsn(**creation_dsn), "-c", - f'CREATE DATABASE "{db_name}"', + f'CREATE DATABASE "{dbname}"', ] ) init_admin_extensions(modname, conninfo) @@ -317,7 +320,7 @@ flavor_set = False for sqlfile in sqlfiles: - logger.debug(f"execute SQL file {sqlfile} db_name={conninfo}") + logger.debug(f"execute SQL file {sqlfile} dbname={conninfo}") subprocess.check_call(psql_command + ["-f", sqlfile]) if flavor is not None and not flavor_set and sqlfile.endswith("-flavor.sql"): diff --git a/swh/core/db/pytest_plugin.py b/swh/core/db/pytest_plugin.py --- a/swh/core/db/pytest_plugin.py +++ b/swh/core/db/pytest_plugin.py @@ -11,9 +11,14 @@ from _pytest.fixtures import FixtureRequest import psycopg2 import pytest -from pytest_postgresql import factories from pytest_postgresql.janitor import DatabaseJanitor, Version +try: + from pytest_postgresql.config import get_config as pytest_postgresql_get_config +except ImportError: + # pytest_postgresql < 3.0.0 + from pytest_postgresql.factories import get_config as pytest_postgresql_get_config + from swh.core.utils import numfile_sortkey as sortkey logger = logging.getLogger(__name__) @@ -34,12 +39,15 @@ user: str, host: str, port: str, - db_name: str, + dbname: str, version: Union[str, float, Version], dump_files: Union[None, str, List[str]] = None, no_truncate_tables: Set[str] = set(), ) -> None: - super().__init__(user, host, port, db_name, version) + super().__init__(user, host, port, dbname, version) + if not hasattr(self, "dbname"): + # pytest_postgresql < 3.0.0 + self.dbname = self.db_name if dump_files is None: self.dump_files = [] elif isinstance(dump_files, str): @@ -51,7 +59,7 @@ def db_setup(self): conninfo = ( - f"host={self.host} user={self.user} port={self.port} dbname={self.db_name}" + f"host={self.host} user={self.user} port={self.port} dbname={self.dbname}" ) for fname in self.dump_files: @@ -74,7 +82,7 @@ """ with psycopg2.connect( - dbname=self.db_name, user=self.user, host=self.host, port=self.port, + dbname=self.dbname, user=self.user, host=self.host, port=self.port, ) as cnx: with cnx.cursor() as cur: cur.execute( @@ -102,20 +110,20 @@ """Initialize db. Create the db if it does not exist. Reset it if it exists.""" with self.cursor() as cur: cur.execute( - "SELECT COUNT(1) FROM pg_database WHERE datname=%s;", (self.db_name,) + "SELECT COUNT(1) FROM pg_database WHERE datname=%s;", (self.dbname,) ) db_exists = cur.fetchone()[0] == 1 if db_exists: cur.execute( "UPDATE pg_database SET datallowconn=true WHERE datname = %s;", - (self.db_name,), + (self.dbname,), ) self.db_reset() return # initialize the inexistent db with self.cursor() as cur: - cur.execute('CREATE DATABASE "{}";'.format(self.db_name)) + cur.execute('CREATE DATABASE "{}";'.format(self.dbname)) self.db_setup() def drop(self): @@ -135,7 +143,7 @@ # specify our version of the DBJanitor we use. def postgresql_fact( process_fixture_name: str, - db_name: Optional[str] = None, + dbname: Optional[str] = None, dump_files: Union[str, List[str]] = "", no_truncate_tables: Set[str] = {"dbversion"}, ): @@ -147,14 +155,14 @@ :rtype: psycopg2.connection :returns: postgresql client """ - config = factories.get_config(request) + config = pytest_postgresql_get_config(request) proc_fixture = request.getfixturevalue(process_fixture_name) pg_host = proc_fixture.host pg_port = proc_fixture.port pg_user = proc_fixture.user pg_options = proc_fixture.options - pg_db = db_name or config["dbname"] + pg_db = dbname or config["dbname"] with SWHDatabaseJanitor( pg_user, pg_host, diff --git a/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py b/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py --- a/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py +++ b/swh/core/db/tests/pytest_plugin/test_pytest_plugin.py @@ -15,14 +15,14 @@ # db with special policy for tables dbversion and people postgres_fun = postgresql_fact( "postgresql_proc", - db_name="fun", + dbname="fun", dump_files=f"{SQL_DIR}/*.sql", no_truncate_tables={"dbversion", "people"}, ) postgres_fun2 = postgresql_fact( "postgresql_proc", - db_name="fun2", + dbname="fun2", dump_files=sorted(glob.glob(f"{SQL_DIR}/*.sql")), no_truncate_tables={"dbversion", "people"}, ) @@ -106,7 +106,7 @@ # db with no special policy for tables truncation, all tables are reset postgres_people = postgresql_fact( "postgresql_proc", - db_name="people", + dbname="people", dump_files=f"{SQL_DIR}/*.sql", no_truncate_tables=set(), ) @@ -160,7 +160,7 @@ # db with no initialization step, an empty db -postgres_no_init = postgresql_fact("postgresql_proc", db_name="something") +postgres_no_init = postgresql_fact("postgresql_proc", dbname="something") def test_smoke_test_db_no_init(postgres_no_init): diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -73,14 +73,14 @@ # We do not want the truncate behavior for those tests test_db = postgresql_fact( - "postgresql_proc", db_name="clidb", no_truncate_tables={"dbversion", "origin"} + "postgresql_proc", dbname="clidb", no_truncate_tables={"dbversion", "origin"} ) @pytest.fixture def swh_db_cli(cli_runner, monkeypatch, test_db): """This initializes a cli_runner and sets the correct environment variable expected by - the cli to run appropriately (when not specifying the --db-name flag) + the cli to run appropriately (when not specifying the --dbname flag) """ db_params = test_db.get_dsn_parameters() @@ -111,11 +111,11 @@ conninfo = craft_conninfo(test_db, "new-db") # This creates the db and installs the necessary admin extensions - result = cli_runner.invoke(swhdb, ["create", module_name, "--db-name", conninfo]) + result = cli_runner.invoke(swhdb, ["create", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # This initializes the schema and data - result = cli_runner.invoke(swhdb, ["init", module_name, "--db-name", conninfo]) + result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" @@ -136,7 +136,7 @@ module_name = "anything" # it's mocked here conninfo = craft_conninfo(test_db, "inexisting-db") - result = cli_runner.invoke(swhdb, ["init", module_name, "--db-name", conninfo]) + result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails because we cannot connect to an inexisting db assert result.exit_code == 1, f"Unexpected output: {result.output}" @@ -152,7 +152,7 @@ module_name = "anything" # it's mocked here conninfo = craft_conninfo(test_db) - result = cli_runner.invoke(swhdb, ["init", module_name, "--db-name", conninfo]) + result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails as the function `public.digest` is not installed, init-admin calls is needed # first (the next tests show such behavior) assert result.exit_code == 1, f"Unexpected output: {result.output}" @@ -167,12 +167,10 @@ module_name = "anything" # it's mocked here conninfo = craft_conninfo(test_db) - result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--db-name", conninfo] - ) + result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" - result = cli_runner.invoke(swhdb, ["init", module_name, "--db-name", conninfo]) + result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise @@ -190,12 +188,12 @@ module_name = "anything" # it's mocked here cli_runner, db_params = swh_db_cli result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" @@ -215,22 +213,22 @@ cli_runner, db_params = swh_db_cli result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init-admin", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( - swhdb, ["init", module_name, "--db-name", db_params["dbname"]] + swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" diff --git a/swh/core/db/tests/test_db.py b/swh/core/db/tests/test_db.py --- a/swh/core/db/tests/test_db.py +++ b/swh/core/db/tests/test_db.py @@ -235,7 +235,7 @@ ] -test_db = postgresql_fact("postgresql_proc", db_name="test-db2") +test_db = postgresql_fact("postgresql_proc", dbname="test-db2") @pytest.fixture