diff --git a/swh/lister/cli.py b/swh/lister/cli.py index 3f59dae..9a9a030 100644 --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -1,129 +1,122 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import click logger = logging.getLogger(__name__) SUPPORTED_LISTERS = ['github', 'gitlab', 'bitbucket', 'debian', 'pypi', 'npm'] @click.command() @click.option( '--db-url', '-d', default='postgres:///lister-gitlab.com', help='SQLAlchemy DB URL; see ' '') # noqa @click.argument('listers', required=1, nargs=-1, type=click.Choice(SUPPORTED_LISTERS + ['all'])) -# help='Listers to act upon') -@click.option('--create-tables', '-t', is_flag=True, default=False, - help='create tables') @click.option('--drop-tables', '-D', is_flag=True, default=False, - help='Drop tables') -@click.option('--with-data', '-f', is_flag=True, default=False, - help='Insert minimum required data') -def cli(db_url, listers, create_tables, drop_tables, with_data): + help='Drop tables before creating the database schema') +def cli(db_url, listers, drop_tables): """Initialize db model according to lister. """ override_conf = { 'lister_db_url': db_url, 'lister': { 'cls': 'local', 'args': {'db': db_url} } } insert_minimum_data = None if 'all' in listers: listers = SUPPORTED_LISTERS for lister in listers: logger.info('Initializing lister %s', lister) if lister == 'github': from .github.models import IndexingModelBase as ModelBase from .github.lister import GitHubLister _lister = GitHubLister( api_baseurl='https://api.github.com', override_config=override_conf) elif lister == 'bitbucket': from .bitbucket.models import IndexingModelBase as ModelBase from .bitbucket.lister import BitBucketLister _lister = BitBucketLister( api_baseurl='https://api.bitbucket.org/2.0', override_config=override_conf) elif lister == 'gitlab': from .gitlab.models import ModelBase from .gitlab.lister import GitLabLister _lister = GitLabLister( api_baseurl='https://gitlab.com/api/v4/', override_config=override_conf) elif lister == 'debian': from .debian.lister import DebianLister ModelBase = DebianLister.MODEL # noqa _lister = DebianLister(override_config=override_conf) def insert_minimum_data(lister): from swh.storage.schemata.distribution import ( Distribution, Area) d = Distribution( name='Debian', type='deb', mirror_uri='http://deb.debian.org/debian/') lister.db_session.add(d) areas = [] for distribution_name in ['stretch']: for area_name in ['main', 'contrib', 'non-free']: areas.append(Area( name='%s/%s' % (distribution_name, area_name), distribution=d, )) lister.db_session.add_all(areas) lister.db_session.commit() elif lister == 'pypi': from .pypi.models import ModelBase from .pypi.lister import PyPILister _lister = PyPILister(override_config=override_conf) elif lister == 'npm': from .npm.models import IndexingModelBase as ModelBase from .npm.models import NpmVisitModel from .npm.lister import NpmLister _lister = NpmLister(override_config=override_conf) if drop_tables: NpmVisitModel.metadata.drop_all(_lister.db_engine) - if create_tables: - NpmVisitModel.metadata.create_all(_lister.db_engine) + NpmVisitModel.metadata.create_all(_lister.db_engine) else: raise ValueError( 'Invalid lister %s: only supported listers are %s' % (lister, SUPPORTED_LISTERS)) if drop_tables: logger.info('Dropping tables for %s', lister) ModelBase.metadata.drop_all(_lister.db_engine) - if create_tables: - logger.info('Creating tables for %s', lister) - ModelBase.metadata.create_all(_lister.db_engine) - - if with_data and insert_minimum_data: - logger.info('Inserting minimal data for %s', lister) - try: - insert_minimum_data(_lister) - except Exception: - logger.warning( - 'Failed to insert minumum data in %s', lister) + logger.info('Creating tables for %s', lister) + ModelBase.metadata.create_all(_lister.db_engine) + + if insert_minimum_data: + logger.info('Inserting minimal data for %s', lister) + try: + insert_minimum_data(_lister) + except Exception: + logger.warning( + 'Failed to insert minumum data in %s', lister) if __name__ == '__main__': cli()