Changeset View
Changeset View
Standalone View
Standalone View
swh/core/cli/db.py
Show All 15 Lines | |||||
from swh.core.cli import CONTEXT_SETTINGS | from swh.core.cli import CONTEXT_SETTINGS | ||||
from swh.core.config import read as config_read | from swh.core.config import read as config_read | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
@click.group(name="db", context_settings=CONTEXT_SETTINGS) | @click.group(name="db", context_settings=CONTEXT_SETTINGS) | ||||
@click.option("--config-file", "-C", default=None, | @click.option( | ||||
"--config-file", | |||||
"-C", | |||||
default=None, | |||||
type=click.Path(exists=True, dir_okay=False), | type=click.Path(exists=True, dir_okay=False), | ||||
help="Configuration file.") | help="Configuration file.", | ||||
) | |||||
@click.pass_context | @click.pass_context | ||||
def db(ctx, config_file): | def db(ctx, config_file): | ||||
"""Software Heritage database generic tools. | """Software Heritage database generic tools. | ||||
""" | """ | ||||
ctx.ensure_object(dict) | ctx.ensure_object(dict) | ||||
if config_file is None: | if config_file is None: | ||||
config_file = environ.get('SWH_CONFIG_FILENAME') | config_file = environ.get("SWH_CONFIG_FILENAME") | ||||
cfg = config_read(config_file) | cfg = config_read(config_file) | ||||
ctx.obj["config"] = cfg | ctx.obj["config"] = cfg | ||||
@db.command(name="init", context_settings=CONTEXT_SETTINGS) | @db.command(name="init", context_settings=CONTEXT_SETTINGS) | ||||
@click.pass_context | @click.pass_context | ||||
def init(ctx): | def init(ctx): | ||||
"""Initialize the database for every Software Heritage module found in the | """Initialize the database for every Software Heritage module found in the | ||||
Show All 27 Lines | def init(ctx): | ||||
""" | """ | ||||
for modname, cfg in ctx.obj["config"].items(): | for modname, cfg in ctx.obj["config"].items(): | ||||
if cfg.get("cls") == "local" and cfg.get("args"): | if cfg.get("cls") == "local" and cfg.get("args"): | ||||
try: | try: | ||||
sqlfiles = get_sql_for_package(modname) | sqlfiles = get_sql_for_package(modname) | ||||
except click.BadParameter: | except click.BadParameter: | ||||
logger.info( | logger.info( | ||||
"Failed to load/find sql initialization files for %s", | "Failed to load/find sql initialization files for %s", modname | ||||
modname) | ) | ||||
if sqlfiles: | if sqlfiles: | ||||
conninfo = cfg["args"]["db"] | conninfo = cfg["args"]["db"] | ||||
for sqlfile in sqlfiles: | for sqlfile in sqlfiles: | ||||
subprocess.check_call( | subprocess.check_call( | ||||
[ | [ | ||||
"psql", | "psql", | ||||
"--quiet", | "--quiet", | ||||
"--no-psqlrc", | "--no-psqlrc", | ||||
"-v", | "-v", | ||||
"ON_ERROR_STOP=1", | "ON_ERROR_STOP=1", | ||||
"-d", | "-d", | ||||
conninfo, | conninfo, | ||||
"-f", | "-f", | ||||
sqlfile, | sqlfile, | ||||
] | ] | ||||
) | ) | ||||
@click.command(context_settings=CONTEXT_SETTINGS) | @click.command(context_settings=CONTEXT_SETTINGS) | ||||
@click.argument('module', nargs=-1, required=True) | @click.argument("module", nargs=-1, required=True) | ||||
@click.option('--db-name', '-d', help='Database name.', | @click.option( | ||||
default='softwareheritage-dev', show_default=True) | "--db-name", | ||||
@click.option('--create-db/--no-create-db', '-C', | "-d", | ||||
help='Attempt to create the database.', | help="Database name.", | ||||
default=False) | default="softwareheritage-dev", | ||||
show_default=True, | |||||
) | |||||
@click.option( | |||||
"--create-db/--no-create-db", | |||||
"-C", | |||||
help="Attempt to create the database.", | |||||
default=False, | |||||
) | |||||
def db_init(module, db_name, create_db): | def db_init(module, db_name, create_db): | ||||
"""Initialise a database for the Software Heritage <module>. By | """Initialise a database for the Software Heritage <module>. By | ||||
default, does not attempt to create the database. | default, does not attempt to create the database. | ||||
Example: | Example: | ||||
swh db-init -d swh-test storage | swh db-init -d swh-test storage | ||||
If you want to specify non-default postgresql connection parameters, | If you want to specify non-default postgresql connection parameters, | ||||
please provide them using standard environment variables. | please provide them using standard environment variables. | ||||
See psql(1) man page (section ENVIRONMENTS) for details. | See psql(1) man page (section ENVIRONMENTS) for details. | ||||
Example: | Example: | ||||
PGPORT=5434 swh db-init indexer | PGPORT=5434 swh db-init indexer | ||||
""" | """ | ||||
# put import statements here so we can keep startup time of the main swh | # put import statements here so we can keep startup time of the main swh | ||||
# command as short as possible | # command as short as possible | ||||
from swh.core.db.tests.db_testing import ( | from swh.core.db.tests.db_testing import ( | ||||
pg_createdb, pg_restore, DB_DUMP_TYPES, | pg_createdb, | ||||
swh_db_version | pg_restore, | ||||
DB_DUMP_TYPES, | |||||
swh_db_version, | |||||
) | ) | ||||
logger.debug('db_init %s dn_name=%s', module, db_name) | logger.debug("db_init %s dn_name=%s", module, db_name) | ||||
dump_files = [] | dump_files = [] | ||||
for modname in module: | for modname in module: | ||||
dump_files.extend(get_sql_for_package(modname)) | dump_files.extend(get_sql_for_package(modname)) | ||||
if create_db: | if create_db: | ||||
# Create the db (or fail silently if already existing) | # Create the db (or fail silently if already existing) | ||||
pg_createdb(db_name, check=False) | pg_createdb(db_name, check=False) | ||||
# Try to retrieve the db version if any | # Try to retrieve the db version if any | ||||
db_version = swh_db_version(db_name) | db_version = swh_db_version(db_name) | ||||
if not db_version: # Initialize the db | if not db_version: # Initialize the db | ||||
dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) | dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files] | ||||
for x in dump_files] | |||||
for dump, dtype in dump_files: | for dump, dtype in dump_files: | ||||
click.secho('Loading {}'.format(dump), fg='yellow') | click.secho("Loading {}".format(dump), fg="yellow") | ||||
pg_restore(db_name, dump, dtype) | pg_restore(db_name, dump, dtype) | ||||
db_version = swh_db_version(db_name) | db_version = swh_db_version(db_name) | ||||
# TODO: Ideally migrate the version from db_version to the latest | # TODO: Ideally migrate the version from db_version to the latest | ||||
# db version | # db version | ||||
click.secho('DONE database is {} version {}'.format(db_name, db_version), | click.secho( | ||||
fg='green', bold=True) | "DONE database is {} version {}".format(db_name, db_version), | ||||
fg="green", | |||||
bold=True, | |||||
) | |||||
def get_sql_for_package(modname): | def get_sql_for_package(modname): | ||||
from importlib import import_module | from importlib import import_module | ||||
from swh.core.utils import numfile_sortkey as sortkey | from swh.core.utils import numfile_sortkey as sortkey | ||||
if not modname.startswith("swh."): | if not modname.startswith("swh."): | ||||
modname = "swh.{}".format(modname) | modname = "swh.{}".format(modname) | ||||
try: | try: | ||||
m = import_module(modname) | m = import_module(modname) | ||||
except ImportError: | except ImportError: | ||||
raise click.BadParameter("Unable to load module {}".format(modname)) | raise click.BadParameter("Unable to load module {}".format(modname)) | ||||
sqldir = path.join(path.dirname(m.__file__), "sql") | sqldir = path.join(path.dirname(m.__file__), "sql") | ||||
if not path.isdir(sqldir): | if not path.isdir(sqldir): | ||||
raise click.BadParameter( | raise click.BadParameter( | ||||
"Module {} does not provide a db schema " | "Module {} does not provide a db schema " "(no sql/ dir)".format(modname) | ||||
"(no sql/ dir)".format(modname)) | ) | ||||
return list(sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey)) | return list(sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey)) |