Changeset View
Standalone View
swh/lister/cli.py
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import logging | |||||
import click | import click | ||||
logger = logging.getLogger(__name__) | |||||
SUPPORTED_LISTERS = ['github', 'gitlab', 'bitbucket', 'debian', 'pypi', 'npm'] | SUPPORTED_LISTERS = ['github', 'gitlab', 'bitbucket', 'debian', 'pypi', 'npm'] | ||||
@click.command() | @click.command() | ||||
@click.option( | @click.option( | ||||
'--db-url', '-d', default='postgres:///lister-gitlab.com', | '--db-url', '-d', default='postgres:///lister-gitlab.com', | ||||
help='SQLAlchemy DB URL; see ' | help='SQLAlchemy DB URL; see ' | ||||
'<http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>') # noqa | '<http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>') # noqa | ||||
@click.option('--lister', required=1, | @click.argument('listers', required=1, nargs=-1, | ||||
vlorentz: Why `-f`? | |||||
Done Inline Actionsbecause -d is already used, and -f for 'fill'. But any better idea is welcome :-) douardda: because -d is already used, and -f for 'fill'. But any better idea is welcome :-) | |||||
Not Done Inline ActionsOh indeed. -i? (and maybe rename the full option to --insert-data?) vlorentz: Oh indeed. `-i`? (and maybe rename the full option to `--insert-data`?) | |||||
Done Inline Actions
I am also tempted to kill this one too. Is there any reason one would NOT want to bootstrap the data? Or at least make it the default behavior... douardda: > -i? (and maybe rename the full option to --insert-data?)
I am also tempted to kill this one… | |||||
Not Done Inline Actionsas far as i remember only the debian lister needed some setup data to actually work. Then, we could rework the other listers with a default function doing nothing. ardumont: as far as i remember only the debian lister needed some setup data to actually work.
Then, we… | |||||
type=click.Choice(SUPPORTED_LISTERS), | type=click.Choice(SUPPORTED_LISTERS + ['all'])) | ||||
help='Lister to act upon') | @click.option('--drop-tables', '-D', is_flag=True, default=False, | ||||
Not Done Inline ActionsNitpick: should be -C for consistency with the next option vlorentz: Nitpick: should be `-C` for consistency with the next option | |||||
Done Inline ActionsIn fact, let's just kill this option! Why on earth would you use this command without this flag on? douardda: In fact, let's just kill this option! Why on earth would you use this command without this flag… | |||||
Not Done Inline ActionsThat was backward compatible with the existing code, that's it, i think. ardumont: That was backward compatible with the existing code, that's it, i think.
I'm fine with removing… | |||||
@click.option('--create-tables', is_flag=True, default=False, | help='Drop tables before creating the database schema') | ||||
help='create tables') | def cli(db_url, listers, drop_tables): | ||||
@click.option('--drop-tables', is_flag=True, default=False, | |||||
help='Drop tables') | |||||
@click.option('--with-data', is_flag=True, default=False, | |||||
help='Insert minimum required data') | |||||
def cli(db_url, lister, create_tables, drop_tables, with_data): | |||||
"""Initialize db model according to lister. | """Initialize db model according to lister. | ||||
""" | """ | ||||
override_conf = {'lister_db_url': db_url} | override_conf = { | ||||
'lister_db_url': db_url, | |||||
'lister': { | |||||
'cls': 'local', | |||||
'args': {'db': db_url} | |||||
} | |||||
} | |||||
insert_minimum_data = None | insert_minimum_data = None | ||||
if 'all' in listers: | |||||
listers = SUPPORTED_LISTERS | |||||
for lister in listers: | |||||
logger.info('Initializing lister %s', lister) | |||||
if lister == 'github': | if lister == 'github': | ||||
from .github.models import IndexingModelBase as ModelBase | from .github.models import IndexingModelBase as ModelBase | ||||
from .github.lister import GitHubLister | from .github.lister import GitHubLister | ||||
_lister = GitHubLister(api_baseurl='https://api.github.com', | _lister = GitHubLister( | ||||
api_baseurl='https://api.github.com', | |||||
override_config=override_conf) | override_config=override_conf) | ||||
elif lister == 'bitbucket': | elif lister == 'bitbucket': | ||||
from .bitbucket.models import IndexingModelBase as ModelBase | from .bitbucket.models import IndexingModelBase as ModelBase | ||||
from .bitbucket.lister import BitBucketLister | from .bitbucket.lister import BitBucketLister | ||||
_lister = BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0', | _lister = BitBucketLister( | ||||
api_baseurl='https://api.bitbucket.org/2.0', | |||||
override_config=override_conf) | override_config=override_conf) | ||||
elif lister == 'gitlab': | elif lister == 'gitlab': | ||||
from .gitlab.models import ModelBase | from .gitlab.models import ModelBase | ||||
from .gitlab.lister import GitLabLister | from .gitlab.lister import GitLabLister | ||||
_lister = GitLabLister(api_baseurl='https://gitlab.com/api/v4/', | _lister = GitLabLister( | ||||
api_baseurl='https://gitlab.com/api/v4/', | |||||
override_config=override_conf) | override_config=override_conf) | ||||
elif lister == 'debian': | elif lister == 'debian': | ||||
from .debian.lister import DebianLister | from .debian.lister import DebianLister | ||||
ModelBase = DebianLister.MODEL # noqa | ModelBase = DebianLister.MODEL # noqa | ||||
_lister = DebianLister(override_config=override_conf) | _lister = DebianLister(override_config=override_conf) | ||||
def insert_minimum_data(lister): | def insert_minimum_data(lister): | ||||
from swh.storage.schemata.distribution import Distribution, Area | from swh.storage.schemata.distribution import ( | ||||
Distribution, Area) | |||||
d = Distribution( | d = Distribution( | ||||
name='Debian', | name='Debian', | ||||
type='deb', | type='deb', | ||||
mirror_uri='http://deb.debian.org/debian/') | mirror_uri='http://deb.debian.org/debian/') | ||||
lister.db_session.add(d) | lister.db_session.add(d) | ||||
areas = [] | areas = [] | ||||
for distribution_name in ['stretch']: | for distribution_name in ['stretch']: | ||||
for area_name in ['main', 'contrib', 'non-free']: | for area_name in ['main', 'contrib', 'non-free']: | ||||
areas.append(Area( | areas.append(Area( | ||||
name='%s/%s' % (distribution_name, area_name), | name='%s/%s' % (distribution_name, area_name), | ||||
distribution=d, | distribution=d, | ||||
)) | )) | ||||
lister.db_session.add_all(areas) | lister.db_session.add_all(areas) | ||||
lister.db_session.commit() | lister.db_session.commit() | ||||
elif lister == 'pypi': | elif lister == 'pypi': | ||||
from .pypi.models import ModelBase | from .pypi.models import ModelBase | ||||
from .pypi.lister import PyPILister | from .pypi.lister import PyPILister | ||||
_lister = PyPILister(override_config=override_conf) | _lister = PyPILister(override_config=override_conf) | ||||
elif lister == 'npm': | elif lister == 'npm': | ||||
from .npm.models import IndexingModelBase as ModelBase | from .npm.models import IndexingModelBase as ModelBase | ||||
from .npm.models import NpmVisitModel | from .npm.models import NpmVisitModel | ||||
from .npm.lister import NpmLister | from .npm.lister import NpmLister | ||||
_lister = NpmLister(override_config=override_conf) | _lister = NpmLister(override_config=override_conf) | ||||
if drop_tables: | if drop_tables: | ||||
NpmVisitModel.metadata.drop_all(_lister.db_engine) | NpmVisitModel.metadata.drop_all(_lister.db_engine) | ||||
if create_tables: | |||||
NpmVisitModel.metadata.create_all(_lister.db_engine) | NpmVisitModel.metadata.create_all(_lister.db_engine) | ||||
else: | else: | ||||
raise ValueError('Only supported listers are %s' % SUPPORTED_LISTERS) | raise ValueError( | ||||
'Invalid lister %s: only supported listers are %s' % | |||||
(lister, SUPPORTED_LISTERS)) | |||||
if drop_tables: | if drop_tables: | ||||
logger.info('Dropping tables for %s', lister) | |||||
ModelBase.metadata.drop_all(_lister.db_engine) | ModelBase.metadata.drop_all(_lister.db_engine) | ||||
if create_tables: | logger.info('Creating tables for %s', lister) | ||||
ModelBase.metadata.create_all(_lister.db_engine) | ModelBase.metadata.create_all(_lister.db_engine) | ||||
if with_data and insert_minimum_data: | if insert_minimum_data: | ||||
logger.info('Inserting minimal data for %s', lister) | |||||
try: | |||||
insert_minimum_data(_lister) | insert_minimum_data(_lister) | ||||
except Exception: | |||||
logger.warning( | |||||
'Failed to insert minumum data in %s', lister) | |||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
cli() | cli() |
Why -f?