diff --git a/swh/lister/cli.py b/swh/lister/cli.py index bb10ac3..964f7ad 100644 --- a/swh/lister/cli.py +++ b/swh/lister/cli.py @@ -1,228 +1,228 @@ # Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import click from swh.core.cli import CONTEXT_SETTINGS logger = logging.getLogger(__name__) SUPPORTED_LISTERS = ['github', 'gitlab', 'bitbucket', 'debian', 'pypi', 'npm', 'phabricator', 'gnu', 'cran', 'cgit', 'packagist'] # Base urls for most listers DEFAULT_BASEURLS = { 'gitlab': 'https://gitlab.com/api/v4/', 'phabricator': 'https://forge.softwareheritage.org', } def get_lister(lister_name, db_url, drop_tables=False, **conf): """Instantiate a lister given its name. Args: lister_name (str): Lister's name db_url (str): Db's service url access conf (dict): Extra configuration (policy, priority for example) Returns: Tuple (instantiated lister, drop_tables function, init schema function, insert minimum data function) """ override_conf = { 'lister': { 'cls': 'local', 'args': {'db': db_url} }, **conf, } # To allow api_baseurl override per lister if 'api_baseurl' in override_conf: api_baseurl = override_conf.pop('api_baseurl') else: api_baseurl = DEFAULT_BASEURLS.get(lister_name) insert_minimum_data_fn = None if lister_name == 'github': from .github.models import IndexingModelBase as ModelBase from .github.lister import GitHubLister _lister = GitHubLister(api_baseurl='https://api.github.com', override_config=override_conf) elif lister_name == 'bitbucket': from .bitbucket.models import IndexingModelBase as ModelBase from .bitbucket.lister import BitBucketLister _lister = BitBucketLister(api_baseurl='https://api.bitbucket.org/2.0', override_config=override_conf) elif lister_name == 'gitlab': from .gitlab.models import ModelBase from .gitlab.lister import GitLabLister _lister = GitLabLister(api_baseurl=api_baseurl, override_config=override_conf) elif lister_name == 'debian': from .debian.lister import DebianLister ModelBase = DebianLister.MODEL # noqa _lister = DebianLister(override_config=override_conf) def insert_minimum_data_fn(lister_name, lister): logger.info('Inserting minimal data for %s', lister_name) from swh.storage.schemata.distribution import ( Distribution, Area) d = Distribution( name='Debian', type='deb', mirror_uri='http://deb.debian.org/debian/') lister.db_session.add(d) areas = [] for distribution_name in ['stretch']: for area_name in ['main', 'contrib', 'non-free']: areas.append(Area( name='%s/%s' % (distribution_name, area_name), distribution=d, )) lister.db_session.add_all(areas) lister.db_session.commit() elif lister_name == 'pypi': from .pypi.models import ModelBase from .pypi.lister import PyPILister _lister = PyPILister(override_config=override_conf) elif lister_name == 'npm': from .npm.models import IndexingModelBase as ModelBase from .npm.models import NpmVisitModel from .npm.lister import NpmLister _lister = NpmLister(override_config=override_conf) def insert_minimum_data_fn(lister_name, lister): logger.info('Inserting minimal data for %s', lister_name) if drop_tables: NpmVisitModel.metadata.drop_all(lister.db_engine) NpmVisitModel.metadata.create_all(lister.db_engine) elif lister_name == 'phabricator': from .phabricator.models import IndexingModelBase as ModelBase from .phabricator.lister import PhabricatorLister - _lister = PhabricatorLister(forge_url=api_baseurl, + _lister = PhabricatorLister(api_baseurl=api_baseurl, override_config=override_conf) elif lister_name == 'gnu': from .gnu.models import ModelBase from .gnu.lister import GNULister _lister = GNULister(override_config=override_conf) elif lister_name == 'cran': from .cran.models import ModelBase from .cran.lister import CRANLister _lister = CRANLister(override_config=override_conf) elif lister_name == 'cgit': from .cgit.models import ModelBase from .cgit.lister import CGitLister _lister = CGitLister(url=api_baseurl, override_config=override_conf) elif lister_name == 'packagist': from .packagist.models import ModelBase # noqa from .packagist.lister import PackagistLister _lister = PackagistLister(override_config=override_conf) else: raise ValueError( 'Invalid lister %s: only supported listers are %s' % (lister_name, SUPPORTED_LISTERS)) drop_table_fn = None if drop_tables: def drop_table_fn(lister_name, lister): logger.info('Dropping tables for %s', lister_name) ModelBase.metadata.drop_all(lister.db_engine) def init_schema_fn(lister_name, lister): logger.info('Creating tables for %s', lister_name) ModelBase.metadata.create_all(lister.db_engine) return _lister, drop_table_fn, init_schema_fn, insert_minimum_data_fn @click.group(name='lister', context_settings=CONTEXT_SETTINGS) @click.pass_context def lister(ctx): '''Software Heritage Lister tools.''' pass @lister.command(name='db-init', context_settings=CONTEXT_SETTINGS) @click.option('--db-url', '-d', default='postgres:///lister', help='SQLAlchemy DB URL; see ' '') # noqa @click.argument('listers', required=1, nargs=-1, type=click.Choice(SUPPORTED_LISTERS + ['all'])) @click.option('--drop-tables', '-D', is_flag=True, default=False, help='Drop tables before creating the database schema') @click.pass_context def cli(ctx, db_url, listers, drop_tables): """Initialize the database model for given listers. """ if 'all' in listers: listers = SUPPORTED_LISTERS for lister_name in listers: logger.info('Initializing lister %s', lister_name) lister, drop_schema_fn, init_schema_fn, insert_minimum_data_fn = \ get_lister(lister_name, db_url, drop_tables=drop_tables) if drop_schema_fn: drop_schema_fn(lister_name, lister) init_schema_fn(lister_name, lister) if insert_minimum_data_fn: insert_minimum_data_fn(lister_name, lister) @lister.command(name='run', context_settings=CONTEXT_SETTINGS, help='Trigger a full listing run for a particular forge ' 'instance. The output of this listing results in ' '"oneshot" tasks in the scheduler db with a priority ' 'defined by the user') @click.option('--db-url', '-d', default='postgres:///lister', help='SQLAlchemy DB URL; see ' '') # noqa @click.option('--lister', '-l', help='Lister to run', type=click.Choice(SUPPORTED_LISTERS)) @click.option('--priority', '-p', default='high', type=click.Choice(['high', 'medium', 'low']), help='Task priority for the listed repositories to ingest') @click.argument('options', nargs=-1) @click.pass_context def run(ctx, db_url, lister, priority, options): from swh.scheduler.cli.utils import parse_options if options: _, kwargs = parse_options(options) else: kwargs = {} override_config = { 'priority': priority, 'policy': 'oneshot', **kwargs, } lister, _, _, _ = get_lister(lister, db_url, **override_config) lister.run() if __name__ == '__main__': cli() diff --git a/swh/lister/npm/models.py b/swh/lister/npm/models.py index 206ae36..5eb8d0d 100644 --- a/swh/lister/npm/models.py +++ b/swh/lister/npm/models.py @@ -1,38 +1,35 @@ # Copyright (C) 2018 the Software Heritage developers # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from sqlalchemy import Column, String, DateTime, Integer, BigInteger, Sequence -from sqlalchemy.ext.declarative import declarative_base -from swh.lister.core.models import IndexingModelBase, ABCSQLMeta - -SQLBase = declarative_base() +from swh.lister.core.models import IndexingModelBase, SQLBase, ABCSQLMeta class NpmVisitModel(SQLBase, metaclass=ABCSQLMeta): """Table to store the npm registry state at the time of a content listing by Software Heritage """ __tablename__ = 'npm_visit' uid = Column(Integer, Sequence('npm_visit_id_seq'), primary_key=True) visit_date = Column(DateTime, nullable=False) doc_count = Column(BigInteger) doc_del_count = Column(BigInteger) update_seq = Column(BigInteger) purge_seq = Column(BigInteger) disk_size = Column(BigInteger) data_size = Column(BigInteger) committed_update_seq = Column(BigInteger) compacted_seq = Column(BigInteger) class NpmModel(IndexingModelBase): """A npm package representation """ __tablename__ = 'npm_repo' uid = Column(String, primary_key=True) indexable = Column(String, index=True)