Changeset View
Changeset View
Standalone View
Standalone View
swh/core/cli/db.py
Show First 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | @click.option( | ||||
default="template1", | default="template1", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
def db_create(module, db_name, template): | def db_create(module, db_name, template): | ||||
"""Create a database for the Software Heritage <module>. | """Create a database for the Software Heritage <module>. | ||||
and potentially execute superuser-level initialization steps. | and potentially execute superuser-level initialization steps. | ||||
Example: | Example:: | ||||
swh db create -d swh-test storage | swh db create -d swh-test storage | ||||
If you want to specify non-default postgresql connection parameters, please | If you want to specify non-default postgresql connection parameters, please | ||||
provide them using standard environment variables or by the mean of a | provide them using standard environment variables or by the mean of a | ||||
properly crafted libpq connection URI. See psql(1) man page (section | properly crafted libpq connection URI. See psql(1) man page (section | ||||
ENVIRONMENTS) for details. | ENVIRONMENTS) for details. | ||||
Note: this command requires a postgresql connection with superuser permissions. | Note: this command requires a postgresql connection with superuser permissions. | ||||
Example: | Example:: | ||||
PGPORT=5434 swh db create indexer | PGPORT=5434 swh db create indexer | ||||
swh db create -d postgresql://superuser:passwd@pghost:5433/swh-storage storage | swh db create -d postgresql://superuser:passwd@pghost:5433/swh-storage storage | ||||
""" | """ | ||||
logger.debug("db_create %s dn_name=%s", module, db_name) | logger.debug("db_create %s dn_name=%s", module, db_name) | ||||
create_database_for_package(module, db_name, template) | create_database_for_package(module, db_name, template) | ||||
@db.command(name="init-admin", context_settings=CONTEXT_SETTINGS) | @db.command(name="init-admin", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("module", required=True) | @click.argument("module", required=True) | ||||
@click.option( | @click.option( | ||||
"--db-name", | "--db-name", | ||||
"-d", | "-d", | ||||
help="Database name.", | help="Database name.", | ||||
default="softwareheritage-dev", | default="softwareheritage-dev", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
def db_init_admin(module: str, db_name: str) -> None: | def db_init_admin(module: str, db_name: str) -> None: | ||||
"""Execute superuser-level initialization steps (e.g pg extensions, admin functions, | """Execute superuser-level initialization steps (e.g pg extensions, admin functions, | ||||
...) | ...) | ||||
Example: | Example:: | ||||
PGPASSWORD=... swh db init-admin -d swh-test scheduler | PGPASSWORD=... swh db init-admin -d swh-test scheduler | ||||
If you want to specify non-default postgresql connection parameters, please | If you want to specify non-default postgresql connection parameters, please | ||||
provide them using standard environment variables or by the mean of a | provide them using standard environment variables or by the mean of a | ||||
properly crafted libpq connection URI. See psql(1) man page (section | properly crafted libpq connection URI. See psql(1) man page (section | ||||
ENVIRONMENTS) for details. | ENVIRONMENTS) for details. | ||||
Note: this command requires a postgresql connection with superuser permissions (e.g | Note: this command requires a postgresql connection with superuser permissions (e.g | ||||
postgres, swh-admin, ...) | postgres, swh-admin, ...) | ||||
Example: | Example:: | ||||
PGPORT=5434 swh db init-admin scheduler | PGPORT=5434 swh db init-admin scheduler | ||||
swh db init-admin -d postgresql://superuser:passwd@pghost:5433/swh-scheduler \ | swh db init-admin -d postgresql://superuser:passwd@pghost:5433/swh-scheduler \ | ||||
scheduler | scheduler | ||||
""" | """ | ||||
logger.debug("db_init_admin %s db_name=%s", module, db_name) | logger.debug("db_init_admin %s db_name=%s", module, db_name) | ||||
init_admin_extensions(module, db_name) | init_admin_extensions(module, db_name) | ||||
@db.command(name="init", context_settings=CONTEXT_SETTINGS) | @db.command(name="init", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("module", required=True) | @click.argument("module", required=True) | ||||
@click.option( | @click.option( | ||||
"--db-name", | "--db-name", | ||||
"-d", | "-d", | ||||
help="Database name.", | help="Database name.", | ||||
default="softwareheritage-dev", | default="softwareheritage-dev", | ||||
show_default=True, | show_default=True, | ||||
) | ) | ||||
@click.option( | @click.option( | ||||
"--flavor", help="Database flavor.", default=None, | "--flavor", help="Database flavor.", default=None, | ||||
) | ) | ||||
def db_init(module, db_name, flavor): | def db_init(module, db_name, flavor): | ||||
"""Initialize a database for the Software Heritage <module>. | """Initialize a database for the Software Heritage <module>. | ||||
Example: | Example:: | ||||
swh db init -d swh-test storage | swh db init -d swh-test storage | ||||
If you want to specify non-default postgresql connection parameters, | If you want to specify non-default postgresql connection parameters, | ||||
please provide them using standard environment variables. | please provide them using standard environment variables. | ||||
See psql(1) man page (section ENVIRONMENTS) for details. | See psql(1) man page (section ENVIRONMENTS) for details. | ||||
Examples: | Examples:: | ||||
PGPORT=5434 swh db init indexer | PGPORT=5434 swh db init indexer | ||||
swh db init -d postgresql://user:passwd@pghost:5433/swh-storage storage | swh db init -d postgresql://user:passwd@pghost:5433/swh-storage storage | ||||
swh db init --flavor read_replica -d swh-storage storage | swh db init --flavor read_replica -d swh-storage storage | ||||
""" | """ | ||||
logger.debug("db_init %s flavor=%s dn_name=%s", module, flavor, db_name) | logger.debug("db_init %s flavor=%s dn_name=%s", module, flavor, db_name) | ||||
initialized, dbversion, dbflavor = populate_database_for_package( | initialized, dbversion, dbflavor = populate_database_for_package( | ||||
module, db_name, flavor | module, db_name, flavor | ||||
) | ) | ||||
Show All 39 Lines | if not path.isdir(sqldir): | ||||
"Module {} does not provide a db schema " "(no sql/ dir)".format(modname) | "Module {} does not provide a db schema " "(no sql/ dir)".format(modname) | ||||
) | ) | ||||
return sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey) | return sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey) | ||||
def populate_database_for_package( | def populate_database_for_package( | ||||
modname: str, conninfo: str, flavor: Optional[str] = None | modname: str, conninfo: str, flavor: Optional[str] = None | ||||
) -> Tuple[bool, int, Optional[str]]: | ) -> Tuple[bool, int, Optional[str]]: | ||||
"""Populate the database, pointed at with `conninfo`, using the SQL files found in | """Populate the database, pointed at with ``conninfo``, | ||||
the package `modname`. | using the SQL files found in the package ``modname``. | ||||
Args: | Args: | ||||
modname: Name of the module of which we're loading the files | modname: Name of the module of which we're loading the files | ||||
conninfo: connection info string for the SQL database | conninfo: connection info string for the SQL database | ||||
flavor: the module-specific flavor which we want to initialize the database under | flavor: the module-specific flavor which we want to initialize the database under | ||||
Returns: | Returns: | ||||
Tuple with three elements: whether the database has been initialized; the current | Tuple with three elements: whether the database has been initialized; the current | ||||
version of the database; if it exists, the flavor of the database. | version of the database; if it exists, the flavor of the database. | ||||
""" | """ | ||||
from swh.core.db.db_utils import swh_db_flavor, swh_db_version | from swh.core.db.db_utils import swh_db_flavor, swh_db_version | ||||
current_version = swh_db_version(conninfo) | current_version = swh_db_version(conninfo) | ||||
if current_version is not None: | if current_version is not None: | ||||
Show All 31 Lines | def init_admin_extensions(modname: str, conninfo: str) -> None: | ||||
sqlfiles = get_sql_for_package(modname) | sqlfiles = get_sql_for_package(modname) | ||||
sqlfiles = [fname for fname in sqlfiles if "-superuser-" in fname] | sqlfiles = [fname for fname in sqlfiles if "-superuser-" in fname] | ||||
execute_sqlfiles(sqlfiles, conninfo) | execute_sqlfiles(sqlfiles, conninfo) | ||||
def create_database_for_package( | def create_database_for_package( | ||||
modname: str, conninfo: str, template: str = "template1" | modname: str, conninfo: str, template: str = "template1" | ||||
): | ): | ||||
"""Create the database pointed at with `conninfo`, and initialize it using | """Create the database pointed at with ``conninfo``, and initialize it using | ||||
-superuser- SQL files found in the package `modname`. | -superuser- SQL files found in the package ``modname``. | ||||
Args: | Args: | ||||
modname: Name of the module of which we're loading the files | modname: Name of the module of which we're loading the files | ||||
conninfo: connection info string or plain database name for the SQL database | conninfo: connection info string or plain database name for the SQL database | ||||
template: the name of the database to connect to and use as template to create | template: the name of the database to connect to and use as template to create | ||||
the new database | the new database | ||||
""" | """ | ||||
import subprocess | import subprocess | ||||
from psycopg2.extensions import make_dsn | from psycopg2.extensions import make_dsn | ||||
# Use the given conninfo string, but with dbname replaced by the template dbname | # Use the given conninfo string, but with dbname replaced by the template dbname | ||||
# for the database creation step | # for the database creation step | ||||
Show All 15 Lines | subprocess.check_call( | ||||
] | ] | ||||
) | ) | ||||
init_admin_extensions(modname, conninfo) | init_admin_extensions(modname, conninfo) | ||||
def execute_sqlfiles( | def execute_sqlfiles( | ||||
sqlfiles: Collection[str], conninfo: str, flavor: Optional[str] = None | sqlfiles: Collection[str], conninfo: str, flavor: Optional[str] = None | ||||
): | ): | ||||
"""Execute a list of SQL files on the database pointed at with `conninfo`. | """Execute a list of SQL files on the database pointed at with ``conninfo``. | ||||
Args: | Args: | ||||
sqlfiles: List of SQL files to execute | sqlfiles: List of SQL files to execute | ||||
conninfo: connection info string for the SQL database | conninfo: connection info string for the SQL database | ||||
flavor: the database flavor to initialize | flavor: the database flavor to initialize | ||||
""" | """ | ||||
import subprocess | import subprocess | ||||
Show All 25 Lines |