Changeset View
Changeset View
Standalone View
Standalone View
swh/core/cli.py
#!/usr/bin/env python3 | #!/usr/bin/env python3 | ||||
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import warnings | import warnings | ||||
warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t | warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t | ||||
from os import path | from os import path | ||||
import glob | import glob | ||||
import click | import click | ||||
from importlib import import_module | from importlib import import_module | ||||
from swh.core.utils import numfile_sortkey as sortkey | from swh.core.utils import numfile_sortkey as sortkey | ||||
from swh.core.tests.db_testing import pg_createdb, pg_restore, DB_DUMP_TYPES | from swh.core.tests.db_testing import ( | ||||
pg_createdb, pg_restore, DB_DUMP_TYPES, | |||||
swh_db_version | |||||
) | |||||
@click.command() | @click.command() | ||||
@click.argument('module', nargs=-1, required=True) | @click.argument('module', nargs=-1, required=True) | ||||
@click.option('--db-name', '-d', help='Database name.', | @click.option('--db-name', '-d', help='Database name.', | ||||
default='softwareheritage-dev', show_default=True) | default='softwareheritage-dev', show_default=True) | ||||
@click.option('--create/--no-create', '-c', | @click.option('--create/--no-create', '-c', | ||||
help='Attempt to create the database', default=True) | help='Attempt to create the database', default=True) | ||||
ardumont: Oh yes, this can be removed, it tries to create the db each time without breaking if it cannot… | |||||
Done Inline Actionsdone ardumont: done | |||||
def db_init(module, db_name=None, create=True): | def db_init(module, db_name=None, create=True): | ||||
"""Initialise a database for the Software Heritage <module>. By | """Initialise a database for the Software Heritage <module>. By | ||||
default, attempts to create the database first. | default, attempts to create the database first. | ||||
Example: | Example: | ||||
swh-db-init storage -d swh-test | swh-db-init storage -d swh-test | ||||
If you want to specify non-default postgresql connection parameters, | If you want to specify non-default postgresql connection parameters, | ||||
please provide them using standard environment variables. | please provide them using standard environment variables. | ||||
See psql(1) man page (section ENVIRONMENTS) for details. | See psql(1) man page (section ENVIRONMENTS) for details. | ||||
Example: | Example: | ||||
PGPORT=5434 swh-db-init indexer -d swh-indexer | PGPORT=5434 swh-db-init indexer -d swh-indexer | ||||
""" | """ | ||||
dump_files = [] | dump_files = [] | ||||
for modname in module: | for modname in module: | ||||
if not modname.startswith('swh.'): | if not modname.startswith('swh.'): | ||||
modname = 'swh.{}'.format(modname) | modname = 'swh.{}'.format(modname) | ||||
try: | try: | ||||
m = import_module(modname) | m = import_module(modname) | ||||
except ImportError: | except ImportError: | ||||
raise click.BadParameter( | raise click.BadParameter( | ||||
'Unable to load module {}'.format(modname)) | 'Unable to load module {}'.format(modname)) | ||||
sqldir = path.join(path.dirname(m.__file__), 'sql') | sqldir = path.join(path.dirname(m.__file__), 'sql') | ||||
if not path.isdir(sqldir): | if not path.isdir(sqldir): | ||||
raise click.BadParameter( | raise click.BadParameter( | ||||
'Module {} does not provide a db schema ' | 'Module {} does not provide a db schema ' | ||||
'(no sql/ dir)'.format(modname)) | '(no sql/ dir)'.format(modname)) | ||||
dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), | dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), | ||||
key=sortkey)) | key=sortkey)) | ||||
if create: | |||||
pg_createdb(db_name) | |||||
# Create the db (or fail silently if already existing) | |||||
pg_createdb(db_name, check=False) | |||||
# Try to retrieve the db version if any | |||||
db_version = swh_db_version(db_name) | |||||
if not db_version: # Initialize the db | |||||
dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) | dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) | ||||
for x in dump_files] | for x in dump_files] | ||||
for dump, dtype in dump_files: | for dump, dtype in dump_files: | ||||
print('dump: %s, type: %s' % (dump, dtype)) | |||||
click.secho('Loading {}'.format(dump), fg='yellow') | click.secho('Loading {}'.format(dump), fg='yellow') | ||||
pg_restore(db_name, dump, dtype) | pg_restore(db_name, dump, dtype) | ||||
Not Done Inline Actionsredundant test vlorentz: redundant test | |||||
Done Inline ActionsIt's to have the db version when we are done migrating the schema. ardumont: It's to have the db version when we are done migrating the schema.
The log at the end of the… | |||||
Done Inline Actionss/migrating/initializing/ ardumont: s/migrating/initializing/ | |||||
Not Done Inline ActionsI know. But since not db_version is always True at this point, you can replace this: if not db_version: db_version = swh_db_version(db_name) with this: db_version = swh_db_version(db_name) vlorentz: I know. But since `not db_version` is always True at this point, you can replace this:
```
if… | |||||
Done Inline ActionsRight! ardumont: Right! | |||||
click.secho('DONE database is {}'.format(db_name), fg='green', bold=True) | if not db_version: | ||||
db_version = swh_db_version(db_name) | |||||
Done Inline ActionsA priori the directive to include the sql upgrade scripts are there so the swh.* pypi service artifact should contain the necessary sql scripts for it. ardumont: A priori the directive to include the sql upgrade scripts are there so the swh.* pypi service… | |||||
# TODO: Ideally migrate the version from db_version to the latest | |||||
# db version | |||||
click.secho('DONE database is {} version {}'.format(db_name, db_version), | |||||
fg='green', bold=True) |
Oh yes, this can be removed, it tries to create the db each time without breaking if it cannot now.