diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 index 25fc474..3166637 --- a/setup.py +++ b/setup.py @@ -1,69 +1,73 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not os.path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.core', description='Software Heritage core utilities', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DCORE/', packages=find_packages(), scripts=[], install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, - entry_points={ - 'console_scripts': ['swh-db-init=swh.core.cli:db_init'], - }, + entry_points=''' + [console_scripts] + swh=swh.core.cli:main + swh-db-init=swh.core.cli.db:db_init + [swh.cli.subcommands] + db-init=swh.core.cli.db:db_init + ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-core', }, ) diff --git a/swh/core/cli/__init__.py b/swh/core/cli/__init__.py new file mode 100644 index 0000000..de82173 --- /dev/null +++ b/swh/core/cli/__init__.py @@ -0,0 +1,40 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import click +import logging +import pkg_resources + +logger = logging.getLogger(__name__) + + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.option('--log-level', '-l', default='INFO', + type=click.Choice(logging._nameToLevel.keys()), + help="Log level (default to INFO)") +@click.pass_context +def swh(ctx, log_level): + """Software Heritage Tool + """ + logger.setLevel(log_level) + ctx.ensure_object(dict) + ctx.obj['log_level'] = logging._nameToLevel[log_level] + + +def main(): + logging.basicConfig() + # load plugins that define cli sub commands + for entry_point in pkg_resources.iter_entry_points('swh.cli.subcommands'): + cmd = entry_point.load() + swh.add_command(cmd, name=entry_point.name) + + return swh(auto_envvar_prefix='SWH') + + +if __name__ == '__main__': + main() diff --git a/swh/core/cli.py b/swh/core/cli/db.py similarity index 80% rename from swh/core/cli.py rename to swh/core/cli/db.py index 0a63ceb..27a1992 100755 --- a/swh/core/cli.py +++ b/swh/core/cli/db.py @@ -1,80 +1,86 @@ #!/usr/bin/env python3 # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import logging import warnings warnings.filterwarnings("ignore") # noqa prevent psycopg from telling us sh*t -from os import path -import glob - import click -from importlib import import_module -from swh.core.utils import numfile_sortkey as sortkey -from swh.core.tests.db_testing import ( - pg_createdb, pg_restore, DB_DUMP_TYPES, - swh_db_version -) + +logger = logging.getLogger(__name__) @click.command() @click.argument('module', nargs=-1, required=True) @click.option('--db-name', '-d', help='Database name.', default='softwareheritage-dev', show_default=True) def db_init(module, db_name=None): """Initialise a database for the Software Heritage . By default, attempts to create the database first. Example: - swh-db-init storage -d swh-test + swh db-init -d swh-test storage If you want to specify non-default postgresql connection parameters, please provide them using standard environment variables. See psql(1) man page (section ENVIRONMENTS) for details. Example: - PGPORT=5434 swh-db-init indexer -d swh-indexer + PGPORT=5434 swh db-init indexer """ + # put import statements here so we can keep startup time of the main swh + # command as short as possible + from os import path + import glob + from importlib import import_module + from swh.core.utils import numfile_sortkey as sortkey + from swh.core.tests.db_testing import ( + pg_createdb, pg_restore, DB_DUMP_TYPES, + swh_db_version + ) + + logger.debug('db_init %s dn_name=%s', module, db_name) dump_files = [] for modname in module: if not modname.startswith('swh.'): modname = 'swh.{}'.format(modname) try: m = import_module(modname) except ImportError: raise click.BadParameter( 'Unable to load module {}'.format(modname)) sqldir = path.join(path.dirname(m.__file__), 'sql') if not path.isdir(sqldir): raise click.BadParameter( 'Module {} does not provide a db schema ' '(no sql/ dir)'.format(modname)) dump_files.extend(sorted(glob.glob(path.join(sqldir, '*.sql')), key=sortkey)) # Create the db (or fail silently if already existing) pg_createdb(db_name, check=False) # Try to retrieve the db version if any db_version = swh_db_version(db_name) if not db_version: # Initialize the db dump_files = [(x, DB_DUMP_TYPES[path.splitext(x)[1]]) for x in dump_files] for dump, dtype in dump_files: click.secho('Loading {}'.format(dump), fg='yellow') pg_restore(db_name, dump, dtype) db_version = swh_db_version(db_name) # TODO: Ideally migrate the version from db_version to the latest # db version click.secho('DONE database is {} version {}'.format(db_name, db_version), fg='green', bold=True)