diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index dd9de8a..66235fe --- a/setup.py +++ b/setup.py @@ -1,78 +1,79 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(*names): requirements = [] for name in names: if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' if not os.path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.core', description='Software Heritage core utilities', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), scripts=[], install_requires=parse_requirements(None, 'swh'), setup_requires=['vcversioner'], extras_require={ 'testing': parse_requirements('test', 'db', 'http'), 'db': parse_requirements('db'), 'http': parse_requirements('http'), }, vcversioner={}, include_package_data=True, entry_points=''' [console_scripts] swh=swh.core.cli:main swh-db-init=swh.core.cli.db:db_init [swh.cli.subcommands] + db=swh.core.cli.db:db db-init=swh.core.cli.db:db_init ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-core', }, ) diff --git a/swh/core/cli/db.py b/swh/core/cli/db.py index 67fa9a3..cee0234 100755 --- a/swh/core/cli/db.py +++ b/swh/core/cli/db.py @@ -1,91 +1,170 @@ #!/usr/bin/env python3 # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import glob import logging +from os import path +import subprocess import warnings + warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t import click from swh.core.cli import CONTEXT_SETTINGS +from swh.core.config import read as config_read logger = logging.getLogger(__name__) +@click.group(name="db", context_settings=CONTEXT_SETTINGS) +@click.option("--config-file", "-C", default=None, + type=click.Path(exists=True, dir_okay=False), + help="Configuration file.") +@click.pass_context +def db(ctx, config_file): + """Software Heritage database generic tools. + """ + ctx.ensure_object(dict) + cfg = config_read(config_file) + ctx.obj["config"] = cfg + + +@db.command(name="init", context_settings=CONTEXT_SETTINGS) +@click.pass_context +def init(ctx): + """Initialize the database for every Software Heritage module found in the + configuration file. For every configuration section in the config file + that: + + 1. has the name of an existing swh package, + 2. has credentials for a local db access, + + it will run the initialization scripts from the swh package against the + given database. + + Example for the config file:: + + \b + storage: + cls: local + args: + db: postgresql:///?service=swh-storage + objstorage: + cls: remote + args: + url: http://swh-objstorage:5003/ + + the command: + + swh db -C /path/to/config.yml init + + will initialize the database for the `storage` section using initialization + scripts from the `swh.storage` package. + """ + + for modname, cfg in ctx.obj["config"].items(): + if cfg.get("cls") == "local" and cfg.get("args"): + try: + sqlfiles = get_sql_for_package(modname) + except click.BadParameter: + logger.info( + "Failed to load/find sql initialization files for %s", + modname) + + if sqlfiles: + conninfo = cfg["args"]["db"] + for sqlfile in sqlfiles: + subprocess.call_call( + [ + "psql", + "--quiet", + "--no-psqlrc", + "-v", + "ON_ERROR_STOP=1", + "-d", + conninfo, + "-f", + sqlfile, + ] + ) + + @click.command(context_settings=CONTEXT_SETTINGS) @click.argument('module', nargs=-1, required=True) @click.option('--db-name', '-d', help='Database name.', default='softwareheritage-dev', show_default=True) @click.option('--create-db/--no-create-db', '-C', help='Attempt to create the database.', default=False) def db_init(module, db_name, create_db): """Initialise a database for the Software Heritage . By default, does not attempt to create the database. Example: swh db-init -d swh-test storage If you want to specify non-default postgresql connection parameters, please provide them using standard environment variables. See psql(1) man page (section ENVIRONMENTS) for details. Example: PGPORT=5434 swh db-init indexer """ # put import statements here so we can keep startup time of the main swh # command as short as possible - from os import path - import glob - from importlib import import_module - from swh.core.utils import numfile_sortkey as sortkey from swh.core.db.tests.db_testing import ( pg_createdb, pg_restore, DB_DUMP_TYPES, swh_db_version ) logger.debug('db_init %s dn_name=%s', module, db_name) dump_files = [] for modname in module: - if not modname.startswith('swh.'): - modname = 'swh.{}'.format(modname) - try: - m = import_module(modname) - except ImportError: - raise click.BadParameter( - 'Unable to load module {}'.format(modname)) - - sqldir = path.join(path.dirname(m.__file__), 'sql') - if not path.isdir(sqldir): - raise click.BadParameter( - 'Module {} does not provide a db schema ' - '(no sql/ dir)'.format(modname)) - dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), - key=sortkey)) + dump_files.extend(get_sql_for_package(modname)) if create_db: # Create the db (or fail silently if already existing) pg_createdb(db_name, check=False) # Try to retrieve the db version if any db_version = swh_db_version(db_name) if not db_version: # Initialize the db dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files] for dump, dtype in dump_files: click.secho('Loading {}'.format(dump), fg='yellow') pg_restore(db_name, dump, dtype) db_version = swh_db_version(db_name) # TODO: Ideally migrate the version from db_version to the latest # db version click.secho('DONE database is {} version {}'.format(db_name, db_version), fg='green', bold=True) + + +def get_sql_for_package(modname): + from importlib import import_module + from swh.core.utils import numfile_sortkey as sortkey + + if not modname.startswith("swh."): + modname = "swh.{}".format(modname) + try: + m = import_module(modname) + except ImportError: + raise click.BadParameter("Unable to load module {}".format(modname)) + + sqldir = path.join(path.dirname(m.__file__), "sql") + if not path.isdir(sqldir): + raise click.BadParameter( + "Module {} does not provide a db schema " + "(no sql/ dir)".format(modname)) + return list(sorted(glob.glob(path.join(sqldir, "*.sql")), key=sortkey)) diff --git a/swh/core/db/tests/test_cli.py b/swh/core/db/tests/test_cli.py new file mode 100644 index 0000000..4a07fdc --- /dev/null +++ b/swh/core/db/tests/test_cli.py @@ -0,0 +1,49 @@ +# + +from click.testing import CliRunner + +from swh.core.cli import swh as swhmain +from swh.core.cli.db import db as swhdb + + +help_msg = '''Usage: swh [OPTIONS] COMMAND [ARGS]... + + Command line interface for Software Heritage. + +Options: + -l, --log-level [NOTSET|DEBUG|INFO|WARNING|ERROR|CRITICAL] + Log level (default to INFO) + -h, --help Show this message and exit. + +Commands: + db Software Heritage database generic tools. +''' + + +def test_swh_help(): + swhmain.add_command(swhdb) + runner = CliRunner() + result = runner.invoke(swhmain, ['-h']) + assert result.exit_code == 0 + assert result.output == help_msg + + +help_db_msg = '''Usage: swh db [OPTIONS] COMMAND [ARGS]... + + Software Heritage database generic tools. + +Options: + -C, --config-file FILE Configuration file. + -h, --help Show this message and exit. + +Commands: + init Initialize the database for every Software Heritage module found in... +''' + + +def test_swh_db_help(): + swhmain.add_command(swhdb) + runner = CliRunner() + result = runner.invoke(swhmain, ['db', '-h']) + assert result.exit_code == 0 + assert result.output == help_db_msg