diff --git a/README.md b/README.md index 0de6256..af79cec 100644 --- a/README.md +++ b/README.md @@ -1,70 +1,71 @@ SWH-lister ============ The Software Heritage Lister is both a library module to permit to centralize lister behaviors, and to provide lister implementations. Actual lister implementations are: - swh-lister-debian - swh-lister-github - swh-lister-bitbucket Licensing ---------- This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. See top-level LICENSE file for the full text of the GNU General Public License along with this program. Dependencies ------------ - python3 - python3-requests - python3-sqlalchemy More details in requirements*.txt Local deployment ----------- ## lister-github 1. git clone under $GHLISTER_ROOT (of your choosing) 2. mkdir ~/.config/swh/ ~/.cache/swh/lister/github.com/ 3. create configuration file ~/.config/swh/lister-github.com.yml 4. Bootstrap the db instance schema -``` sh -$ createdb lister-github.com -$ python3 -m swh.lister.cli --db-url postgres:///lister-github.com github --createdb -``` + $ createdb lister-github + $ python3 -m swh.lister.cli --db-url postgres:///lister-github \ + --lister github \ + --create-tables + Configuration file samples ------------------------- ## github cat ~/.config/swh/lister-github.com.yml # see http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls lister_db_url: postgres:///lister-github.com credentials: [] cache_responses: True cache_dir: /home/zack/.cache/swh/lister/github.com storage: cls: remote args: url: http://localhost:5002/ diff --git a/swh/lister/cli.py b/swh/lister/cli.py index 82f8934..c6644dc 100644 --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -1,98 +1,59 @@ # Copyright (C) 2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - - -@click.group(context_settings=CONTEXT_SETTINGS) +@click.command() @click.option( '--db-url', '-d', default='postgres:///lister-gitlab.com', help='SQLAlchemy DB URL; see ' '') # noqa -@click.pass_context -def cli(ctx, db_url): +@click.option('--lister', required=1, + type=click.Choice(['github', 'gitlab', 'bitbucket']), + help='Lister to act upon') +@click.option('--create-tables', is_flag=True, default=False, + help='create tables') +@click.option('--drop-tables', is_flag=True, default=False, + help='Drop db') +def cli(db_url, lister, create_tables, drop_tables): """Initialize db model according to lister. """ - config = {} - if db_url: - config['db_url'] = db_url - ctx.obj = config - - -@cli.command('github') -@click.option('--createdb', is_flag=True, default=False, - help='create db') -@click.option('--dropdb', is_flag=True, default=False, - help='Drop db') -@click.pass_context -def github(ctx, createdb, dropdb): - from .github import models - from .github.lister import GitHubLister - - override_conf = {'lister_db_url': ctx.obj['db_url']} - - lister = GitHubLister(lister_name='github.com', - api_baseurl='https://api.github.com', - override_config=override_conf) - - if dropdb: - models.ModelBase.metadata.drop_all(lister.db_engine) - - if createdb: - models.ModelBase.metadata.create_all(lister.db_engine) - - -@cli.command('gitlab') -@click.option('--createdb', is_flag=True, default=False, - help='create db') -@click.option('--dropdb', is_flag=True, default=False, - help='Drop db') -@click.pass_context -def gitlab(ctx, createdb, dropdb): - from .gitlab import models - from .gitlab.lister import GitlabLister - - override_conf = {'lister_db_url': ctx.obj['db_url']} - - lister = GitlabLister(lister_name='gitlab.com', - api_baseurl='https://gitlab.com/api/v4/', - override_config=override_conf) - - if dropdb: - models.ModelBase.metadata.drop_all(lister.db_engine) - - if createdb: - models.ModelBase.metadata.create_all(lister.db_engine) - - -@cli.command('bitbucket') -@click.option('--createdb', is_flag=True, default=False, - help='create db') -@click.option('--dropdb', is_flag=True, default=False, - help='Drop db') -@click.pass_context -def bitbucket(ctx, createdb, dropdb): - from .bitbucket import models - from .bitbucket.lister import BitBucketLister - - override_conf = {'lister_db_url': ctx.obj['db_url']} - - lister = BitBucketLister(lister_name='bitbucket.com', - api_baseurl='https://api.bitbucket.org/2.0', - override_config=override_conf) - - if dropdb: - models.ModelBase.metadata.drop_all(lister.db_engine) - - if createdb: - models.ModelBase.metadata.create_all(lister.db_engine) + supported_listers = ['github', 'gitlab', 'bitbucket'] + override_conf = {'lister_db_url': db_url} + + if lister == 'github': + from .github import models + from .github.lister import GitHubLister + + _lister = GitHubLister(lister_name='github.com', + api_baseurl='https://api.github.com', + override_config=override_conf) + elif lister == 'bitbucket': + from .bitbucket import models + from .bitbucket.lister import BitBucketLister + _lister = BitBucketLister(lister_name='bitbucket.com', + api_baseurl='https://api.bitbucket.org/2.0', + override_config=override_conf) + + elif lister == 'gitlab': + from .gitlab import models + from .gitlab.lister import GitLabLister + _lister = GitLabLister(lister_name='gitlab.com', + api_baseurl='https://gitlab.com/api/v4/', + override_config=override_conf) + else: + raise ValueError('Only supported listers are %s' % supported_listers) + + if drop_tables: + models.ModelBase.metadata.drop_all(_lister.db_engine) + + if create_tables: + models.ModelBase.metadata.create_all(_lister.db_engine) if __name__ == '__main__': cli()