diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -306,6 +306,69 @@ click.echo(f"{version} [{tstamp}] {desc}") +@db.command(name="upgrade", context_settings=CONTEXT_SETTINGS) +@click.argument("module", required=True) +@click.option( + "--to-version", + type=int, + help="Upgrade up to version VERSION", + metavar="VERSION", + default=None, +) +@click.pass_context +def db_upgrade(ctx, module, to_version): + """Upgrade the database for given module. + + Example:: + + swh db upgrade storage + + """ + from swh.core.db.db_utils import get_database_info, import_swhmodule, swh_db_upgrade + + # use the db cnx from the config file; the expected config entry is the + # given module name + cfg = ctx.obj["config"].get(module, {}) + dbname = get_dburl_from_config(cfg) + + if not dbname: + raise click.BadParameter( + "Missing the postgresql connection configuration. Either fix your " + "configuration file or use the --dbname option." + ) + + logger.debug("db_version dbname=%s", dbname) + + db_module, db_version, db_flavor = get_database_info(dbname) + if db_module is None: + click.secho( + "WARNING the database does not have a dbmodule table.", fg="red", bold=True + ) + db_module = module + assert db_module == module, f"{db_module} (in the db) != {module} (given)" + + # instantiate the data source to retrieve the current (expected) db version + datastore_factory = getattr(import_swhmodule(db_module), "get_datastore", None) + if not datastore_factory: + raise click.UsageError( + "You cannot use this command on old-style datastore " f"backend {db_module}" + ) + datastore = datastore_factory(**cfg) + ds_version = datastore.get_current_version() + if to_version is None: + to_version = ds_version + else: + to_version = min(to_version, ds_version) + + new_db_version = swh_db_upgrade(dbname, module, to_version) + click.secho(f"Migration to version {new_db_version} done", fg="green") + if new_db_version < ds_version: + click.secho( + f"Warning: migration was not complete: the current version is {ds_version}", + fg="yellow", + ) + + def get_dburl_from_config(cfg): if cfg.get("cls") != "postgresql": raise click.BadParameter( diff --git a/swh/core/db/db_utils.py b/swh/core/db/db_utils.py --- a/swh/core/db/db_utils.py +++ b/swh/core/db/db_utils.py @@ -149,6 +149,66 @@ return None +def swh_db_upgrade( + conninfo: str, modname: str, to_version: Optional[int] = None +) -> int: + """ + Args: + db_or_conninfo: A database connection, or a database connection info string + sql_dir: directory path where to find upgrade sql files + """ + + if to_version is None: + to_version = 99999999 + + dn_module, db_version, db_flavor = get_database_info(conninfo) + if db_version is None: + raise ValueError("Unable to retrieve the current version of the database") + + sqlfiles = [ + fname + for fname in get_sql_for_package(modname, upgrade=True) + if db_version < int(path.splitext(path.basename(fname))[0]) <= to_version + ] + + for sqlfile in sqlfiles: + new_version = int(path.splitext(path.basename(sqlfile))[0]) + logger.info("Executing migration script {sqlfile}") + if db_version is not None and (new_version - db_version) > 1: + logger.error( + f"There are missing migration steps between {db_version} and " + f"{new_version}. It might be expected but it most unlikely is not. " + "Will stop here." + ) + return db_version + + execute_sqlfiles([sqlfile], conninfo, db_flavor) + + # check if the db version has been updated by the upgrade script + db_version = swh_db_version(conninfo) + assert db_version is not None + if db_version == new_version: + # nothing to do, upgrade script did the job + pass + elif db_version == new_version - 1: + # it has not (new style), so do it + swh_set_db_version( + conninfo, + new_version, + desc=f"Upgraded to version {new_version} using {sqlfile}", + ) + db_version = swh_db_version(conninfo) + else: + # upgrade script did it wrong + logger.error( + f"The upgrade script {sqlfile} did not update the dbversion table " + f"consistently ({db_version} vs. expected {new_version}). " + "Will stop migration here. Please check your migration scripts." + ) + return db_version + return new_version + + def swh_db_module(db_or_conninfo: Union[str, pgconnection]) -> Optional[str]: """Retrieve the swh module used to create the database. @@ -397,11 +457,13 @@ return m -def get_sql_for_package(modname): +def get_sql_for_package(modname, upgrade=False): m = import_swhmodule(modname) if m is None: raise ValueError(f"Module {modname} cannot be loaded") sqldir = path.join(path.dirname(m.__file__), "sql") + if upgrade: + sqldir /= "upgrades" if not path.isdir(sqldir): raise ValueError( "Module {} does not provide a db schema " "(no sql/ dir)".format(modname) diff --git a/swh/core/db/tests/conftest.py b/swh/core/db/tests/conftest.py --- a/swh/core/db/tests/conftest.py +++ b/swh/core/db/tests/conftest.py @@ -38,9 +38,11 @@ cli_runner.invoke(swhdb, ["init", module_name, "--dbname", conninfo]) """ - def get_sql_for_package_mock(modname): + def get_sql_for_package_mock(modname, upgrade=False): if modname.startswith("test."): sqldir = modname.split(".", 1)[1] + if upgrade: + sqldir += "/upgrades" return sorted( glob.glob(os.path.join(datadir, sqldir, "*.sql")), key=sortkey ) diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py --- a/swh/core/db/tests/test_cli.py +++ b/swh/core/db/tests/test_cli.py @@ -263,3 +263,84 @@ cur.execute("select * from origin") origins = cur.fetchall() assert len(origins) == 1 + + +def test_cli_swh_db_upgrade_new_api( + cli_runner, postgresql, mock_package_sql, mocker, tmp_path +): + """Create a db then initializing it should be ok for a "new style" datastore + + """ + module_name = "test.cli_new" + + from unittest.mock import MagicMock + + from swh.core.db.db_utils import import_swhmodule, swh_db_version + + current_version = 1 + + def import_swhmodule_mock(modname): + if modname.startswith("test."): + + def get_datastore(cls, **kw): + # XXX probably not the best way of doing this... + return MagicMock(get_current_version=lambda: current_version) + + return MagicMock(name=modname, get_datastore=get_datastore) + + return import_swhmodule(modname) + + mocker.patch("swh.core.db.db_utils.import_swhmodule", import_swhmodule_mock) + conninfo = craft_conninfo(postgresql) + + # This initializes the schema and data + cfgfile = tmp_path / "config.yml" + cfgfile.write_text( + f""" +{module_name}: + cls: postgresql + db: {conninfo} +""" + ) + result = cli_runner.invoke(swhdb, ["init-admin", module_name, "--dbname", conninfo]) + assert result.exit_code == 0, f"Unexpected output: {result.output}" + result = cli_runner.invoke(swhdb, ["-C", cfgfile, "init", module_name]) + + import traceback + + assert ( + result.exit_code == 0 + ), f"Unexpected output: {traceback.print_tb(result.exc_info[2])}" + + assert swh_db_version(conninfo) == 1 + + # the upgrade should not do anything because the datastore does advertise + # version 1 + result = cli_runner.invoke(swhdb, ["-C", cfgfile, "upgrade", module_name]) + assert swh_db_version(conninfo) == 1 + + # advertize current version as 3, a simple upgrade should get us there, but + # no further + current_version = 3 + result = cli_runner.invoke(swhdb, ["-C", cfgfile, "upgrade", module_name]) + assert swh_db_version(conninfo) == 3 + + # an attempt to go further should not do anything + result = cli_runner.invoke( + swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 5] + ) + assert swh_db_version(conninfo) == 3 + # an attempt to go lower should not do anything + result = cli_runner.invoke( + swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 2] + ) + assert swh_db_version(conninfo) == 3 + + # advertize current version as 6, an upgrade with --to-version 4 should + # sticj to the given version 4 and no further + current_version = 6 + result = cli_runner.invoke( + swhdb, ["-C", cfgfile, "upgrade", module_name, "--to-version", 4] + ) + assert swh_db_version(conninfo) == 4 + assert "migration was not complete" in result.output diff --git a/swh/core/db/tests/test_db_utils.py b/swh/core/db/tests/test_db_utils.py --- a/swh/core/db/tests/test_db_utils.py +++ b/swh/core/db/tests/test_db_utils.py @@ -1,10 +1,17 @@ from datetime import datetime, timedelta +from os import path import pytest from swh.core.cli.db import db as swhdb from swh.core.db import BaseDb -from swh.core.db.db_utils import get_database_info, now, swh_db_module, swh_db_versions +from swh.core.db.db_utils import ( + get_database_info, + now, + swh_db_module, + swh_db_upgrade, + swh_db_versions, +) from .test_cli import craft_conninfo @@ -71,3 +78,41 @@ if version > 10: assert desc == f"Upgrade to version {version}" assert (now() - ts) < timedelta(seconds=1) + + +@pytest.mark.parametrize("module", ["test.cli_new"]) +def test_db_utils_upgrade(cli_runner, postgresql, mock_package_sql, module, datadir): + """Check swh_db_upgrade + + """ + conninfo = craft_conninfo(postgresql) + result = cli_runner.invoke(swhdb, ["init-admin", module, "--dbname", conninfo]) + assert result.exit_code == 0, f"Unexpected output: {result.output}" + result = cli_runner.invoke(swhdb, ["init", module, "--dbname", conninfo]) + assert result.exit_code == 0, f"Unexpected output: {result.output}" + + new_version = swh_db_upgrade(conninfo, module) + assert new_version == 6 + + versions = swh_db_versions(conninfo) + # get rid of dates to ease checking + versions = [(v[0], v[2]) for v in versions] + assert versions[-1] == (1, "DB initialization") + sqlbasedir = path.join(datadir, module.split(".", 1)[1], "upgrades") + + assert versions[1:-1] == [ + (i, f"Upgraded to version {i} using {sqlbasedir}/{i:03d}.sql") + for i in range(5, 1, -1) + ] + assert versions[0] == (6, "Updated version from upgrade script") + + cnx = BaseDb.connect(conninfo) + with cnx.transaction() as cur: + cur.execute("select url from origin where url like 'version%'") + result = cur.fetchall() + assert result == [("version%03d" % i,) for i in range(2, 7)] + cur.execute( + "select url from origin where url = 'this should never be executed'" + ) + result = cur.fetchall() + assert not result