diff --git a/setup.py b/setup.py index 5f24715..2ad98e6 100755 --- a/setup.py +++ b/setup.py @@ -1,75 +1,75 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.loader.core', description='Software Heritage Base Loader', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DLDBASE', packages=find_packages(), # packages's modules scripts=[], # scripts to package install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, entry_points=''' [swh.workers] loader.archive=swh.loader.package.archive:register loader.debian=swh.loader.package.debian:register loader.deposit=swh.loader.package.deposit:register loader.npm=swh.loader.package.npm:register loader.pypi=swh.loader.package.pypi:register [swh.cli.subcommands] - loader=swh.loader.cli:run + loader=swh.loader.cli:loader ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-loader-core', }, ) diff --git a/swh/loader/cli.py b/swh/loader/cli.py index 613042b..12a0e6b 100644 --- a/swh/loader/cli.py +++ b/swh/loader/cli.py @@ -1,68 +1,90 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import inspect import logging import click import pkg_resources from typing import Any from swh.core.cli import CONTEXT_SETTINGS from swh.scheduler.cli.utils import parse_options logger = logging.getLogger(__name__) LOADERS = {entry_point.name.split('.', 1)[1]: entry_point for entry_point in pkg_resources.iter_entry_points('swh.workers') if entry_point.name.split('.', 1)[0] == 'loader'} SUPPORTED_LOADERS = list(LOADERS) def get_loader(name: str, **kwargs) -> Any: """Given a loader name, instantiate it. Args: name: Loader's name kwargs: Configuration dict (url...) Returns: An instantiated loader """ if name not in LOADERS: raise ValueError( 'Invalid loader %s: only supported loaders are %s' % (name, SUPPORTED_LOADERS)) registry_entry = LOADERS[name].load()() logger.debug(f'registry: {registry_entry}') loader_cls = registry_entry['loader'] logger.debug(f'loader class: {loader_cls}') return loader_cls(**kwargs) -@click.command(name='run', context_settings=CONTEXT_SETTINGS) -@click.option('--type', '-t', help='Loader to run', - type=click.Choice(SUPPORTED_LOADERS)) -@click.option('--url', '-u', default=None, - help="Origin url to load") -@click.argument('options', nargs=-1) +@click.group(name='loader', context_settings=CONTEXT_SETTINGS) @click.pass_context -def run(ctx, type, url, options): +def loader(ctx): """Loader cli tools - Load an origin from its url with loader - """ + pass + + +@loader.command(name='run', context_settings=CONTEXT_SETTINGS) +@click.argument('type', + type=click.Choice(SUPPORTED_LOADERS)) +@click.argument('url') +@click.argument('options', nargs=-1) +@click.pass_context +def run(ctx, type, url, options): + """Ingest with loader the origin located at """ (_, kw) = parse_options(options) logger.debug(f'kw: {kw}') loader = get_loader(type, url=url, **kw) result = loader.load() click.echo(result) + + +@loader.command(name='list', context_settings=CONTEXT_SETTINGS) +@click.argument('type', default='all', + type=click.Choice(['all'] + SUPPORTED_LOADERS)) +@click.pass_context +def list(ctx, type): + """List supported loaders and optionally their arguments""" + if type == 'all': + loaders = ', '.join(SUPPORTED_LOADERS) + click.echo(f'Supported loaders: {loaders}') + else: + registry_entry = LOADERS[type].load()() + loader_cls = registry_entry['loader'] + doc = inspect.getdoc(loader_cls).strip() + signature = inspect.signature(loader_cls) + click.echo(f"Loader: {doc}\nsignature: {signature}") diff --git a/swh/loader/tests/test_cli.py b/swh/loader/tests/test_cli.py index dcee86a..6db5573 100644 --- a/swh/loader/tests/test_cli.py +++ b/swh/loader/tests/test_cli.py @@ -1,90 +1,102 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest -from swh.loader.cli import run, get_loader, SUPPORTED_LOADERS +from swh.loader.cli import run, list, get_loader, SUPPORTED_LOADERS from swh.loader.package.loader import PackageLoader from click.testing import CliRunner def test_get_loader_wrong_input(swh_config): """Unsupported loader should raise """ loader_type = 'unknown' assert loader_type not in SUPPORTED_LOADERS with pytest.raises(ValueError, match='Invalid loader'): get_loader(loader_type, url='db-url') def test_get_loader(swh_config): """Instantiating a supported loader should be ok """ loader_input = { 'archive': { 'url': 'some-url', 'artifacts': [], }, 'debian': { 'url': 'some-url', 'date': 'something', 'packages': [], }, 'deposit': { 'url': 'some-url', 'deposit_id': 1, }, 'npm': { 'url': 'https://www.npmjs.com/package/onepackage', }, 'pypi': { 'url': 'some-url', }, } for loader_type, kwargs in loader_input.items(): loader = get_loader(loader_type, **kwargs) assert isinstance(loader, PackageLoader) -help_msg = """Usage: run [OPTIONS] [OPTIONS]... +run_help_msg = """Usage: run [OPTIONS] [archive|debian|deposit|npm|pypi] URL [OPTIONS]... - Loader cli tools - - Load an origin from its url with loader + Ingest with loader the origin located at Options: - -t, --type [archive|debian|deposit|npm|pypi] - Loader to run - -u, --url TEXT Origin url to load - -h, --help Show this message and exit. -""" + -h, --help Show this message and exit. +""" # noqa def test_run_help(swh_config): """Help message should be ok """ runner = CliRunner() result = runner.invoke(run, ['-h']) + assert result.exit_code == 0 - assert result.output.startswith(help_msg) + assert result.output.startswith(run_help_msg) def test_run_pypi(mocker, swh_config): """Triggering a load should be ok """ mock_loader = mocker.patch('swh.loader.package.pypi.loader.PyPILoader') runner = CliRunner() - result = runner.invoke(run, [ - '--type', 'pypi', - '--url', 'https://some-url' - ]) + result = runner.invoke(run, ['pypi', 'https://some-url']) assert result.exit_code == 0 mock_loader.assert_called_once_with(url='https://some-url') # constructor + + +list_help_msg = """Usage: list [OPTIONS] [[all|archive|debian|deposit|npm|pypi]] + + List supported loaders and optionally their arguments + +Options: + -h, --help Show this message and exit. +""" # noqa + + +def test_list_help(mocker, swh_config): + """Triggering a load should be ok + + """ + runner = CliRunner() + result = runner.invoke(list, ['--help']) + assert result.exit_code == 0 + assert result.output.startswith(list_help_msg)