diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,3 +1,4 @@ swh.core >= 0.0.75 swh.model >= 0.0.18 swh.storage >= 0.0.153 +swh.scheduler diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -57,6 +57,8 @@ loader.deposit=swh.loader.package.deposit:register loader.npm=swh.loader.package.npm:register loader.pypi=swh.loader.package.pypi:register + [swh.cli.subcommands] + loader=swh.loader.cli:run ''', classifiers=[ "Programming Language :: Python :: 3", diff --git a/swh/loader/cli.py b/swh/loader/cli.py new file mode 100644 --- /dev/null +++ b/swh/loader/cli.py @@ -0,0 +1,66 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import logging + +import click +import pkg_resources + +from swh.core.cli import CONTEXT_SETTINGS +from swh.scheduler.cli.utils import parse_options + + +logger = logging.getLogger(__name__) + + +LOADERS = {entry_point.name.split('.', 1)[1]: entry_point + for entry_point in pkg_resources.iter_entry_points('swh.workers') + if entry_point.name.split('.', 1)[0] == 'loader'} + + +SUPPORTED_LOADERS = list(LOADERS) + + +def get_loader(name: str, **kwargs): + """Given a loader name, instantiate it. + + Args: + name (str): Lister's name + conf (dict): Configuration dict (lister db cnx, policy, priority...) + + Returns: + An instantiated loader + + """ + if name not in LOADERS: + raise ValueError( + 'Invalid lister %s: only supported listers are %s' % + (name, SUPPORTED_LOADERS)) + + registry_entry = LOADERS[name].load()() + logger.debug(f'registry: {registry_entry}') + loader_cls = registry_entry['loader'] + logger.debug(f'loader class: {loader_cls}') + return loader_cls(**kwargs) + + +@click.command(name='run', context_settings=CONTEXT_SETTINGS) +@click.option('--type', '-t', help='Loader to run', + type=click.Choice(SUPPORTED_LOADERS)) +@click.option('--url', '-u', default=None, + help="Origin url to load") +@click.argument('options', nargs=-1) +@click.pass_context +def run(ctx, type, url, options): + """Loader cli tools + + Load an origin from its url with loader + + """ + (_, kw) = parse_options(options) + logger.debug(f'kw: {kw}') + loader = get_loader(type, url=url, **kw) + result = loader.load() + click.echo(result) diff --git a/swh/loader/package/archive/__init__.py b/swh/loader/package/archive/__init__.py --- a/swh/loader/package/archive/__init__.py +++ b/swh/loader/package/archive/__init__.py @@ -9,6 +9,8 @@ def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import ArchiveLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': ArchiveLoader, } diff --git a/swh/loader/package/debian/__init__.py b/swh/loader/package/debian/__init__.py --- a/swh/loader/package/debian/__init__.py +++ b/swh/loader/package/debian/__init__.py @@ -9,6 +9,8 @@ def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import DebianLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': DebianLoader, } diff --git a/swh/loader/package/deposit/__init__.py b/swh/loader/package/deposit/__init__.py --- a/swh/loader/package/deposit/__init__.py +++ b/swh/loader/package/deposit/__init__.py @@ -9,6 +9,8 @@ def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import DepositLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': DepositLoader, } diff --git a/swh/loader/package/npm/__init__.py b/swh/loader/package/npm/__init__.py --- a/swh/loader/package/npm/__init__.py +++ b/swh/loader/package/npm/__init__.py @@ -9,6 +9,8 @@ def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import NpmLoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': NpmLoader, } diff --git a/swh/loader/package/pypi/__init__.py b/swh/loader/package/pypi/__init__.py --- a/swh/loader/package/pypi/__init__.py +++ b/swh/loader/package/pypi/__init__.py @@ -9,6 +9,8 @@ def register() -> Mapping[str, Any]: """Register the current worker module's definition""" + from .loader import PyPILoader return { 'task_modules': [f'{__name__}.tasks'], + 'loader': PyPILoader, }