Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/cli.py
# Copyright (C) 2018-2019 The Software Heritage developers | # Copyright (C) 2018-2019 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import os | import os | ||||
import logging | import logging | ||||
import pkg_resources | |||||
from copy import deepcopy | from copy import deepcopy | ||||
from importlib import import_module | from importlib import import_module | ||||
import click | import click | ||||
from sqlalchemy import create_engine | from sqlalchemy import create_engine | ||||
from swh.core.cli import CONTEXT_SETTINGS | from swh.core.cli import CONTEXT_SETTINGS | ||||
from swh.scheduler import get_scheduler | from swh.scheduler import get_scheduler | ||||
from swh.scheduler.task import SWHTask | from swh.scheduler.task import SWHTask | ||||
from swh.lister import get_lister, SUPPORTED_LISTERS, LISTERS | |||||
from swh.lister.core.models import initialize | from swh.lister.core.models import initialize | ||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
LISTERS = {entry_point.name.split('.', 1)[1]: entry_point | |||||
for entry_point in pkg_resources.iter_entry_points('swh.workers') | |||||
if entry_point.name.split('.', 1)[0] == 'lister'} | |||||
SUPPORTED_LISTERS = list(LISTERS) | |||||
# the key in this dict is the suffix used to match new task-type to be added. | # the key in this dict is the suffix used to match new task-type to be added. | ||||
# For example for a task which function name is "list_gitlab_full', the default | # For example for a task which function name is "list_gitlab_full', the default | ||||
# value used when inserting a new task-type in the scheduler db will be the one | # value used when inserting a new task-type in the scheduler db will be the one | ||||
# under the 'full' key below (because it matches xxx_full). | # under the 'full' key below (because it matches xxx_full). | ||||
DEFAULT_TASK_TYPE = { | DEFAULT_TASK_TYPE = { | ||||
'full': { # for tasks like 'list_xxx_full()' | 'full': { # for tasks like 'list_xxx_full()' | ||||
'default_interval': '90 days', | 'default_interval': '90 days', | ||||
'min_interval': '90 days', | 'min_interval': '90 days', | ||||
'max_interval': '90 days', | 'max_interval': '90 days', | ||||
'backoff_factor': 1 | 'backoff_factor': 1 | ||||
}, | }, | ||||
'*': { # value if not suffix matches | '*': { # value if not suffix matches | ||||
'default_interval': '1 day', | 'default_interval': '1 day', | ||||
'min_interval': '1 day', | 'min_interval': '1 day', | ||||
'max_interval': '1 day', | 'max_interval': '1 day', | ||||
'backoff_factor': 1 | 'backoff_factor': 1 | ||||
}, | }, | ||||
} | } | ||||
def get_lister(lister_name, db_url=None, **conf): | |||||
"""Instantiate a lister given its name. | |||||
Args: | |||||
lister_name (str): Lister's name | |||||
conf (dict): Configuration dict (lister db cnx, policy, priority...) | |||||
Returns: | |||||
Tuple (instantiated lister, drop_tables function, init schema function, | |||||
insert minimum data function) | |||||
""" | |||||
if lister_name not in LISTERS: | |||||
raise ValueError( | |||||
'Invalid lister %s: only supported listers are %s' % | |||||
(lister_name, SUPPORTED_LISTERS)) | |||||
if db_url: | |||||
conf['lister'] = {'cls': 'local', 'args': {'db': db_url}} | |||||
registry_entry = LISTERS[lister_name].load()() | |||||
lister_cls = registry_entry['lister'] | |||||
lister = lister_cls(override_config=conf) | |||||
return lister | |||||
@click.group(name='lister', context_settings=CONTEXT_SETTINGS) | @click.group(name='lister', context_settings=CONTEXT_SETTINGS) | ||||
@click.option('--config-file', '-C', default=None, | @click.option('--config-file', '-C', default=None, | ||||
type=click.Path(exists=True, dir_okay=False,), | type=click.Path(exists=True, dir_okay=False,), | ||||
help="Configuration file.") | help="Configuration file.") | ||||
@click.option('--db-url', '-d', default=None, | @click.option('--db-url', '-d', default=None, | ||||
help='SQLAlchemy DB URL; see ' | help='SQLAlchemy DB URL; see ' | ||||
'<http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>') # noqa | '<http://docs.sqlalchemy.org/en/latest/core/engines.html#database-urls>') # noqa | ||||
@click.pass_context | @click.pass_context | ||||
▲ Show 20 Lines • Show All 155 Lines • Show Last 20 Lines |