Changeset View
Changeset View
Standalone View
Standalone View
swh/core/cli/db.py
Show All 37 Lines | if config_file is None: | ||||
config_file = environ.get("SWH_CONFIG_FILENAME") | config_file = environ.get("SWH_CONFIG_FILENAME") | ||||
cfg = config_read(config_file) | cfg = config_read(config_file) | ||||
ctx.obj["config"] = cfg | ctx.obj["config"] = cfg | ||||
@db.command(name="create", context_settings=CONTEXT_SETTINGS) | @db.command(name="create", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("module", required=True) | @click.argument("module", required=True) | ||||
@click.option( | @click.option( | ||||
"--dbname", | |||||
"--db-name", | "--db-name", | ||||
"-d", | "-d", | ||||
help="Database name.", | help="Database name.", | ||||
default="softwareheritage-dev", | default="softwareheritage-dev", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
@click.option( | @click.option( | ||||
"--template", | "--template", | ||||
"-T", | "-T", | ||||
help="Template database from which to build this database.", | help="Template database from which to build this database.", | ||||
default="template1", | default="template1", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
def db_create(module, db_name, template): | def db_create(module, dbname, template): | ||||
"""Create a database for the Software Heritage <module>. | """Create a database for the Software Heritage <module>. | ||||
and potentially execute superuser-level initialization steps. | and potentially execute superuser-level initialization steps. | ||||
Example:: | Example:: | ||||
swh db create -d swh-test storage | swh db create -d swh-test storage | ||||
If you want to specify non-default postgresql connection parameters, please | If you want to specify non-default postgresql connection parameters, please | ||||
provide them using standard environment variables or by the mean of a | provide them using standard environment variables or by the mean of a | ||||
properly crafted libpq connection URI. See psql(1) man page (section | properly crafted libpq connection URI. See psql(1) man page (section | ||||
ENVIRONMENTS) for details. | ENVIRONMENTS) for details. | ||||
Note: this command requires a postgresql connection with superuser permissions. | Note: this command requires a postgresql connection with superuser permissions. | ||||
Example:: | Example:: | ||||
PGPORT=5434 swh db create indexer | PGPORT=5434 swh db create indexer | ||||
swh db create -d postgresql://superuser:passwd@pghost:5433/swh-storage storage | swh db create -d postgresql://superuser:passwd@pghost:5433/swh-storage storage | ||||
""" | """ | ||||
logger.debug("db_create %s dn_name=%s", module, db_name) | logger.debug("db_create %s dn_name=%s", module, dbname) | ||||
create_database_for_package(module, db_name, template) | create_database_for_package(module, dbname, template) | ||||
@db.command(name="init-admin", context_settings=CONTEXT_SETTINGS) | @db.command(name="init-admin", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("module", required=True) | @click.argument("module", required=True) | ||||
@click.option( | @click.option( | ||||
"--dbname", | |||||
"--db-name", | "--db-name", | ||||
"-d", | "-d", | ||||
help="Database name.", | help="Database name.", | ||||
default="softwareheritage-dev", | default="softwareheritage-dev", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
def db_init_admin(module: str, db_name: str) -> None: | def db_init_admin(module: str, dbname: str) -> None: | ||||
"""Execute superuser-level initialization steps (e.g pg extensions, admin functions, | """Execute superuser-level initialization steps (e.g pg extensions, admin functions, | ||||
...) | ...) | ||||
Example:: | Example:: | ||||
PGPASSWORD=... swh db init-admin -d swh-test scheduler | PGPASSWORD=... swh db init-admin -d swh-test scheduler | ||||
If you want to specify non-default postgresql connection parameters, please | If you want to specify non-default postgresql connection parameters, please | ||||
provide them using standard environment variables or by the mean of a | provide them using standard environment variables or by the mean of a | ||||
properly crafted libpq connection URI. See psql(1) man page (section | properly crafted libpq connection URI. See psql(1) man page (section | ||||
ENVIRONMENTS) for details. | ENVIRONMENTS) for details. | ||||
Note: this command requires a postgresql connection with superuser permissions (e.g | Note: this command requires a postgresql connection with superuser permissions (e.g | ||||
postgres, swh-admin, ...) | postgres, swh-admin, ...) | ||||
Example:: | Example:: | ||||
PGPORT=5434 swh db init-admin scheduler | PGPORT=5434 swh db init-admin scheduler | ||||
swh db init-admin -d postgresql://superuser:passwd@pghost:5433/swh-scheduler \ | swh db init-admin -d postgresql://superuser:passwd@pghost:5433/swh-scheduler \ | ||||
scheduler | scheduler | ||||
""" | """ | ||||
logger.debug("db_init_admin %s db_name=%s", module, db_name) | logger.debug("db_init_admin %s dbname=%s", module, dbname) | ||||
init_admin_extensions(module, db_name) | init_admin_extensions(module, dbname) | ||||
@db.command(name="init", context_settings=CONTEXT_SETTINGS) | @db.command(name="init", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("module", required=True) | @click.argument("module", required=True) | ||||
@click.option( | @click.option( | ||||
"--dbname", | |||||
"--db-name", | "--db-name", | ||||
"-d", | "-d", | ||||
help="Database name.", | help="Database name.", | ||||
default="softwareheritage-dev", | default="softwareheritage-dev", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
@click.option( | @click.option( | ||||
"--flavor", help="Database flavor.", default=None, | "--flavor", help="Database flavor.", default=None, | ||||
) | ) | ||||
def db_init(module, db_name, flavor): | def db_init(module, dbname, flavor): | ||||
"""Initialize a database for the Software Heritage <module>. | """Initialize a database for the Software Heritage <module>. | ||||
Example:: | Example:: | ||||
swh db init -d swh-test storage | swh db init -d swh-test storage | ||||
If you want to specify non-default postgresql connection parameters, | If you want to specify non-default postgresql connection parameters, | ||||
please provide them using standard environment variables. | please provide them using standard environment variables. | ||||
See psql(1) man page (section ENVIRONMENTS) for details. | See psql(1) man page (section ENVIRONMENTS) for details. | ||||
Examples:: | Examples:: | ||||
PGPORT=5434 swh db init indexer | PGPORT=5434 swh db init indexer | ||||
swh db init -d postgresql://user:passwd@pghost:5433/swh-storage storage | swh db init -d postgresql://user:passwd@pghost:5433/swh-storage storage | ||||
swh db init --flavor read_replica -d swh-storage storage | swh db init --flavor read_replica -d swh-storage storage | ||||
""" | """ | ||||
logger.debug("db_init %s flavor=%s dn_name=%s", module, flavor, db_name) | logger.debug("db_init %s flavor=%s dbname=%s", module, flavor, dbname) | ||||
initialized, dbversion, dbflavor = populate_database_for_package( | initialized, dbversion, dbflavor = populate_database_for_package( | ||||
module, db_name, flavor | module, dbname, flavor | ||||
) | ) | ||||
# TODO: Ideally migrate the version from db_version to the latest | # TODO: Ideally migrate the version from db_version to the latest | ||||
# db version | # db version | ||||
click.secho( | click.secho( | ||||
"DONE database for {} {}{} at version {}".format( | "DONE database for {} {}{} at version {}".format( | ||||
module, | module, | ||||
▲ Show 20 Lines • Show All 104 Lines • ▼ Show 20 Lines | ): | ||||
""" | """ | ||||
import subprocess | import subprocess | ||||
from psycopg2.extensions import make_dsn | from psycopg2.extensions import make_dsn | ||||
# Use the given conninfo string, but with dbname replaced by the template dbname | # Use the given conninfo string, but with dbname replaced by the template dbname | ||||
# for the database creation step | # for the database creation step | ||||
creation_dsn = parse_dsn_or_dbname(conninfo) | creation_dsn = parse_dsn_or_dbname(conninfo) | ||||
db_name = creation_dsn["dbname"] | dbname = creation_dsn["dbname"] | ||||
creation_dsn["dbname"] = template | creation_dsn["dbname"] = template | ||||
logger.debug("db_create db_name=%s (from %s)", db_name, template) | logger.debug("db_create dbname=%s (from %s)", dbname, template) | ||||
subprocess.check_call( | subprocess.check_call( | ||||
[ | [ | ||||
"psql", | "psql", | ||||
"--quiet", | "--quiet", | ||||
"--no-psqlrc", | "--no-psqlrc", | ||||
"-v", | "-v", | ||||
"ON_ERROR_STOP=1", | "ON_ERROR_STOP=1", | ||||
"-d", | "-d", | ||||
make_dsn(**creation_dsn), | make_dsn(**creation_dsn), | ||||
"-c", | "-c", | ||||
f'CREATE DATABASE "{db_name}"', | f'CREATE DATABASE "{dbname}"', | ||||
] | ] | ||||
) | ) | ||||
init_admin_extensions(modname, conninfo) | init_admin_extensions(modname, conninfo) | ||||
def execute_sqlfiles( | def execute_sqlfiles( | ||||
sqlfiles: Collection[str], conninfo: str, flavor: Optional[str] = None | sqlfiles: Collection[str], conninfo: str, flavor: Optional[str] = None | ||||
): | ): | ||||
Show All 13 Lines | psql_command = [ | ||||
"-v", | "-v", | ||||
"ON_ERROR_STOP=1", | "ON_ERROR_STOP=1", | ||||
"-d", | "-d", | ||||
conninfo, | conninfo, | ||||
] | ] | ||||
flavor_set = False | flavor_set = False | ||||
for sqlfile in sqlfiles: | for sqlfile in sqlfiles: | ||||
logger.debug(f"execute SQL file {sqlfile} db_name={conninfo}") | logger.debug(f"execute SQL file {sqlfile} dbname={conninfo}") | ||||
subprocess.check_call(psql_command + ["-f", sqlfile]) | subprocess.check_call(psql_command + ["-f", sqlfile]) | ||||
if flavor is not None and not flavor_set and sqlfile.endswith("-flavor.sql"): | if flavor is not None and not flavor_set and sqlfile.endswith("-flavor.sql"): | ||||
logger.debug("Setting database flavor %s", flavor) | logger.debug("Setting database flavor %s", flavor) | ||||
query = f"insert into dbflavor (flavor) values ('{flavor}')" | query = f"insert into dbflavor (flavor) values ('{flavor}')" | ||||
subprocess.check_call(psql_command + ["-c", query]) | subprocess.check_call(psql_command + ["-c", query]) | ||||
flavor_set = True | flavor_set = True | ||||
if flavor is not None and not flavor_set: | if flavor is not None and not flavor_set: | ||||
logger.warn( | logger.warn( | ||||
"Asked for flavor %s, but module does not support database flavors", flavor, | "Asked for flavor %s, but module does not support database flavors", flavor, | ||||
) | ) |