diff --git a/MANIFEST.in b/MANIFEST.in index 9f03021..9199b48 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,7 +1,8 @@ include Makefile include conftest.py include requirements*.txt include version.txt recursive-include swh py.typed +recursive-include swh/core/db/tests/data/ * recursive-include swh/core/tests/data/ * recursive-include swh/core/tests/fixture/data/ * diff --git a/swh/core/db/tests/data/cli/1-schema.sql b/swh/core/db/tests/data/cli/30-schema.sql similarity index 100% rename from swh/core/db/tests/data/cli/1-schema.sql rename to swh/core/db/tests/data/cli/30-schema.sql diff --git a/swh/core/db/tests/data/cli/3-func.sql b/swh/core/db/tests/data/cli/40-funcs.sql similarity index 100% rename from swh/core/db/tests/data/cli/3-func.sql rename to swh/core/db/tests/data/cli/40-funcs.sql diff --git a/swh/core/db/tests/data/cli/4-data.sql b/swh/core/db/tests/data/cli/50-data.sql similarity index 100% rename from swh/core/db/tests/data/cli/4-data.sql rename to swh/core/db/tests/data/cli/50-data.sql diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py index d9d71da..312f2a2 100644 --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -1,240 +1,257 @@ # Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import glob from os import path from click.testing import CliRunner import pytest from swh.core.cli.db import db as swhdb from swh.core.db import BaseDb from swh.core.db.pytest_plugin import postgresql_fact from swh.core.tests.test_cli import assert_section_contains @pytest.fixture def cli_runner(): return CliRunner() def test_cli_swh_help(swhmain, cli_runner): swhmain.add_command(swhdb) result = cli_runner.invoke(swhmain, ["-h"]) assert result.exit_code == 0 assert_section_contains( result.output, "Commands", "db Software Heritage database generic tools." ) help_db_snippets = ( ( "Usage", ( "Usage: swh db [OPTIONS] COMMAND [ARGS]...", "Software Heritage database generic tools.", ), ), ( "Commands", ( "create Create a database for the Software Heritage .", "init Initialize a database for the Software Heritage .", "init-admin Execute superuser-level initialization steps", ), ), ) def test_cli_swh_db_help(swhmain, cli_runner): swhmain.add_command(swhdb) result = cli_runner.invoke(swhmain, ["db", "-h"]) assert result.exit_code == 0 for section, snippets in help_db_snippets: for snippet in snippets: assert_section_contains(result.output, section, snippet) @pytest.fixture() def mock_package_sql(mocker, datadir): """This bypasses the module manipulation to only returns the data test files. + For a given module `test.mod`, look for sql files in the directory `data/mod/*.sql`. + + Typical usage:: + + def test_xxx(cli_runner, mock_package_sql): + conninfo = craft_conninfo(test_db, "new-db") + module_name = "test.cli" + # the command below will use sql scripts from swh/core/db/tests/data/cli/*.sql + cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) """ + from swh.core.db.db_utils import get_sql_for_package from swh.core.utils import numfile_sortkey as sortkey - mock_sql_files = mocker.patch("swh.core.db.db_utils.get_sql_for_package") - sql_files = sorted(glob.glob(path.join(datadir, "cli", "*.sql")), key=sortkey) - mock_sql_files.return_value = sql_files + def get_sql_for_package_mock(modname): + if modname.startswith("test."): + sqldir = modname.split(".", 1)[1] + return sorted(glob.glob(path.join(datadir, sqldir, "*.sql")), key=sortkey) + return get_sql_for_package(modname) + + mock_sql_files = mocker.patch( + "swh.core.db.db_utils.get_sql_for_package", get_sql_for_package_mock + ) + return mock_sql_files # We do not want the truncate behavior for those tests test_db = postgresql_fact( "postgresql_proc", dbname="clidb", no_truncate_tables={"dbversion", "origin"} ) @pytest.fixture def swh_db_cli(cli_runner, monkeypatch, test_db): """This initializes a cli_runner and sets the correct environment variable expected by the cli to run appropriately (when not specifying the --dbname flag) """ db_params = test_db.get_dsn_parameters() monkeypatch.setenv("PGHOST", db_params["host"]) monkeypatch.setenv("PGUSER", db_params["user"]) monkeypatch.setenv("PGPORT", db_params["port"]) return cli_runner, db_params def craft_conninfo(test_db, dbname=None) -> str: """Craft conninfo string out of the test_db object. This also allows to override the dbname.""" db_params = test_db.get_dsn_parameters() if dbname: params = copy.deepcopy(db_params) params["dbname"] = dbname else: params = db_params return "postgresql://{user}@{host}:{port}/{dbname}".format(**params) def test_cli_swh_db_create_and_init_db(cli_runner, test_db, mock_package_sql): """Create a db then initializing it should be ok """ - module_name = "something" + module_name = "test.cli" conninfo = craft_conninfo(test_db, "new-db") # This creates the db and installs the necessary admin extensions result = cli_runner.invoke(swhdb, ["create", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # This initializes the schema and data result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin value in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, installed during db creation step) with BaseDb.connect(conninfo).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_fail_without_creation_first( cli_runner, test_db, mock_package_sql ): """Init command on an inexisting db cannot work """ - module_name = "anything" # it's mocked here + module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(test_db, "inexisting-db") result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails because we cannot connect to an inexisting db assert result.exit_code == 1, f"Unexpected output: {result.output}" def test_cli_swh_db_initialization_fail_without_extension( cli_runner, test_db, mock_package_sql ): """Init command cannot work without privileged extension. In this test, the schema needs privileged extension to work. """ - module_name = "anything" # it's mocked here + module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(test_db) result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) # Fails as the function `public.digest` is not installed, init-admin calls is needed # first (the next tests show such behavior) assert result.exit_code == 1, f"Unexpected output: {result.output}" def test_cli_swh_db_initialization_works_with_flags( cli_runner, test_db, mock_package_sql ): """Init commands with carefully crafted libpq conninfo works """ - module_name = "anything" # it's mocked here + module_name = "test.cli" # it's mocked here conninfo = craft_conninfo(test_db) result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(test_db.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_with_env(swh_db_cli, mock_package_sql, test_db): """Init commands with standard environment variables works """ - module_name = "anything" # it's mocked here + module_name = "test.cli" # it's mocked here cli_runner, db_params = swh_db_cli result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(test_db.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 def test_cli_swh_db_initialization_idempotent(swh_db_cli, mock_package_sql, test_db): """Multiple runs of the init commands are idempotent """ - module_name = "anything" # mocked + module_name = "test.cli" # mocked cli_runner, db_params = swh_db_cli result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init-admin", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" result = cli_runner.invoke( swhdb, ["init", module_name, "--dbname", db_params["dbname"]] ) assert result.exit_code == 0, f"Unexpected output: {result.output}" # the origin values in the scripts uses a hash function (which implementation wise # uses a function from the pgcrypt extension, init-admin calls installs it) with BaseDb.connect(test_db.dsn).cursor() as cur: cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1