diff --git a/setup.py b/setup.py
index 5f24715..2ad98e6 100755
--- a/setup.py
+++ b/setup.py
@@ -1,75 +1,75 @@
#!/usr/bin/env python3
# Copyright (C) 2015-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from setuptools import setup, find_packages
from os import path
from io import open
here = path.abspath(path.dirname(__file__))
# Get the long description from the README file
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
def parse_requirements(name=None):
if name:
reqf = 'requirements-%s.txt' % name
else:
reqf = 'requirements.txt'
requirements = []
if not path.exists(reqf):
return requirements
with open(reqf) as f:
for line in f.readlines():
line = line.strip()
if not line or line.startswith('#'):
continue
requirements.append(line)
return requirements
setup(
name='swh.loader.core',
description='Software Heritage Base Loader',
long_description=long_description,
long_description_content_type='text/markdown',
author='Software Heritage developers',
author_email='swh-devel@inria.fr',
url='https://forge.softwareheritage.org/diffusion/DLDBASE',
packages=find_packages(), # packages's modules
scripts=[], # scripts to package
install_requires=parse_requirements() + parse_requirements('swh'),
setup_requires=['vcversioner'],
extras_require={'testing': parse_requirements('test')},
vcversioner={},
include_package_data=True,
entry_points='''
[swh.workers]
loader.archive=swh.loader.package.archive:register
loader.debian=swh.loader.package.debian:register
loader.deposit=swh.loader.package.deposit:register
loader.npm=swh.loader.package.npm:register
loader.pypi=swh.loader.package.pypi:register
[swh.cli.subcommands]
- loader=swh.loader.cli:run
+ loader=swh.loader.cli:loader
''',
classifiers=[
"Programming Language :: Python :: 3",
"Intended Audience :: Developers",
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
"Operating System :: OS Independent",
"Development Status :: 5 - Production/Stable",
],
project_urls={
'Bug Reports': 'https://forge.softwareheritage.org/maniphest',
'Funding': 'https://www.softwareheritage.org/donate',
'Source': 'https://forge.softwareheritage.org/source/swh-loader-core',
},
)
diff --git a/swh/loader/cli.py b/swh/loader/cli.py
index 613042b..12a0e6b 100644
--- a/swh/loader/cli.py
+++ b/swh/loader/cli.py
@@ -1,68 +1,90 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import inspect
import logging
import click
import pkg_resources
from typing import Any
from swh.core.cli import CONTEXT_SETTINGS
from swh.scheduler.cli.utils import parse_options
logger = logging.getLogger(__name__)
LOADERS = {entry_point.name.split('.', 1)[1]: entry_point
for entry_point in pkg_resources.iter_entry_points('swh.workers')
if entry_point.name.split('.', 1)[0] == 'loader'}
SUPPORTED_LOADERS = list(LOADERS)
def get_loader(name: str, **kwargs) -> Any:
"""Given a loader name, instantiate it.
Args:
name: Loader's name
kwargs: Configuration dict (url...)
Returns:
An instantiated loader
"""
if name not in LOADERS:
raise ValueError(
'Invalid loader %s: only supported loaders are %s' %
(name, SUPPORTED_LOADERS))
registry_entry = LOADERS[name].load()()
logger.debug(f'registry: {registry_entry}')
loader_cls = registry_entry['loader']
logger.debug(f'loader class: {loader_cls}')
return loader_cls(**kwargs)
-@click.command(name='run', context_settings=CONTEXT_SETTINGS)
-@click.option('--type', '-t', help='Loader to run',
- type=click.Choice(SUPPORTED_LOADERS))
-@click.option('--url', '-u', default=None,
- help="Origin url to load")
-@click.argument('options', nargs=-1)
+@click.group(name='loader', context_settings=CONTEXT_SETTINGS)
@click.pass_context
-def run(ctx, type, url, options):
+def loader(ctx):
"""Loader cli tools
- Load an origin from its url with loader <name>
-
"""
+ pass
+
+
+@loader.command(name='run', context_settings=CONTEXT_SETTINGS)
+@click.argument('type',
+ type=click.Choice(SUPPORTED_LOADERS))
+@click.argument('url')
+@click.argument('options', nargs=-1)
+@click.pass_context
+def run(ctx, type, url, options):
+ """Ingest with loader <type> the origin located at <url>"""
(_, kw) = parse_options(options)
logger.debug(f'kw: {kw}')
loader = get_loader(type, url=url, **kw)
result = loader.load()
click.echo(result)
+
+
+@loader.command(name='list', context_settings=CONTEXT_SETTINGS)
+@click.argument('type', default='all',
+ type=click.Choice(['all'] + SUPPORTED_LOADERS))
+@click.pass_context
+def list(ctx, type):
+ """List supported loaders and optionally their arguments"""
+ if type == 'all':
+ loaders = ', '.join(SUPPORTED_LOADERS)
+ click.echo(f'Supported loaders: {loaders}')
+ else:
+ registry_entry = LOADERS[type].load()()
+ loader_cls = registry_entry['loader']
+ doc = inspect.getdoc(loader_cls).strip()
+ signature = inspect.signature(loader_cls)
+ click.echo(f"Loader: {doc}\nsignature: {signature}")
diff --git a/swh/loader/tests/test_cli.py b/swh/loader/tests/test_cli.py
index dcee86a..6db5573 100644
--- a/swh/loader/tests/test_cli.py
+++ b/swh/loader/tests/test_cli.py
@@ -1,90 +1,102 @@
# Copyright (C) 2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import pytest
-from swh.loader.cli import run, get_loader, SUPPORTED_LOADERS
+from swh.loader.cli import run, list, get_loader, SUPPORTED_LOADERS
from swh.loader.package.loader import PackageLoader
from click.testing import CliRunner
def test_get_loader_wrong_input(swh_config):
"""Unsupported loader should raise
"""
loader_type = 'unknown'
assert loader_type not in SUPPORTED_LOADERS
with pytest.raises(ValueError, match='Invalid loader'):
get_loader(loader_type, url='db-url')
def test_get_loader(swh_config):
"""Instantiating a supported loader should be ok
"""
loader_input = {
'archive': {
'url': 'some-url',
'artifacts': [],
},
'debian': {
'url': 'some-url',
'date': 'something',
'packages': [],
},
'deposit': {
'url': 'some-url',
'deposit_id': 1,
},
'npm': {
'url': 'https://www.npmjs.com/package/onepackage',
},
'pypi': {
'url': 'some-url',
},
}
for loader_type, kwargs in loader_input.items():
loader = get_loader(loader_type, **kwargs)
assert isinstance(loader, PackageLoader)
-help_msg = """Usage: run [OPTIONS] [OPTIONS]...
+run_help_msg = """Usage: run [OPTIONS] [archive|debian|deposit|npm|pypi] URL [OPTIONS]...
- Loader cli tools
-
- Load an origin from its url with loader <name>
+ Ingest with loader <type> the origin located at <url>
Options:
- -t, --type [archive|debian|deposit|npm|pypi]
- Loader to run
- -u, --url TEXT Origin url to load
- -h, --help Show this message and exit.
-"""
+ -h, --help Show this message and exit.
+""" # noqa
def test_run_help(swh_config):
"""Help message should be ok
"""
runner = CliRunner()
result = runner.invoke(run, ['-h'])
+
assert result.exit_code == 0
- assert result.output.startswith(help_msg)
+ assert result.output.startswith(run_help_msg)
def test_run_pypi(mocker, swh_config):
"""Triggering a load should be ok
"""
mock_loader = mocker.patch('swh.loader.package.pypi.loader.PyPILoader')
runner = CliRunner()
- result = runner.invoke(run, [
- '--type', 'pypi',
- '--url', 'https://some-url'
- ])
+ result = runner.invoke(run, ['pypi', 'https://some-url'])
assert result.exit_code == 0
mock_loader.assert_called_once_with(url='https://some-url') # constructor
+
+
+list_help_msg = """Usage: list [OPTIONS] [[all|archive|debian|deposit|npm|pypi]]
+
+ List supported loaders and optionally their arguments
+
+Options:
+ -h, --help Show this message and exit.
+""" # noqa
+
+
+def test_list_help(mocker, swh_config):
+ """Triggering a load should be ok
+
+ """
+ runner = CliRunner()
+ result = runner.invoke(list, ['--help'])
+ assert result.exit_code == 0
+ assert result.output.startswith(list_help_msg)