Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/cli.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import inspect | # WARNING: do not import unnecessary things here to keep cli startup time under | ||||
# control | |||||
import logging | import logging | ||||
import click | import click | ||||
import iso8601 | |||||
import pkg_resources | import pkg_resources | ||||
from typing import Any | from typing import Any | ||||
from swh.core.cli import CONTEXT_SETTINGS | from swh.core.cli import CONTEXT_SETTINGS | ||||
from swh.scheduler.cli.utils import parse_options | |||||
logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
LOADERS = { | LOADERS = { | ||||
entry_point.name.split(".", 1)[1]: entry_point | entry_point.name.split(".", 1)[1]: entry_point | ||||
for entry_point in pkg_resources.iter_entry_points("swh.workers") | for entry_point in pkg_resources.iter_entry_points("swh.workers") | ||||
Show All 39 Lines | |||||
@loader.command(name="run", context_settings=CONTEXT_SETTINGS) | @loader.command(name="run", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | @click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | ||||
@click.argument("url") | @click.argument("url") | ||||
@click.argument("options", nargs=-1) | @click.argument("options", nargs=-1) | ||||
@click.pass_context | @click.pass_context | ||||
def run(ctx, type, url, options): | def run(ctx, type, url, options): | ||||
"""Ingest with loader <type> the origin located at <url>""" | """Ingest with loader <type> the origin located at <url>""" | ||||
import iso8601 | |||||
from swh.scheduler.cli.utils import parse_options | |||||
(_, kw) = parse_options(options) | (_, kw) = parse_options(options) | ||||
logger.debug(f"kw: {kw}") | logger.debug(f"kw: {kw}") | ||||
visit_date = kw.get("visit_date") | visit_date = kw.get("visit_date") | ||||
if visit_date and isinstance(visit_date, str): | if visit_date and isinstance(visit_date, str): | ||||
visit_date = iso8601.parse_date(visit_date) | visit_date = iso8601.parse_date(visit_date) | ||||
kw["visit_date"] = visit_date | kw["visit_date"] = visit_date | ||||
loader = get_loader(type, url=url, **kw) | loader = get_loader(type, url=url, **kw) | ||||
result = loader.load() | result = loader.load() | ||||
click.echo(result) | click.echo(result) | ||||
@loader.command(name="list", context_settings=CONTEXT_SETTINGS) | @loader.command(name="list", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("type", default="all", type=click.Choice(["all"] + SUPPORTED_LOADERS)) | @click.argument("type", default="all", type=click.Choice(["all"] + SUPPORTED_LOADERS)) | ||||
@click.pass_context | @click.pass_context | ||||
def list(ctx, type): | def list(ctx, type): | ||||
"""List supported loaders and optionally their arguments""" | """List supported loaders and optionally their arguments""" | ||||
import inspect | |||||
if type == "all": | if type == "all": | ||||
loaders = ", ".join(SUPPORTED_LOADERS) | loaders = ", ".join(SUPPORTED_LOADERS) | ||||
click.echo(f"Supported loaders: {loaders}") | click.echo(f"Supported loaders: {loaders}") | ||||
else: | else: | ||||
registry_entry = LOADERS[type].load()() | registry_entry = LOADERS[type].load()() | ||||
loader_cls = registry_entry["loader"] | loader_cls = registry_entry["loader"] | ||||
doc = inspect.getdoc(loader_cls).strip() | doc = inspect.getdoc(loader_cls).strip() | ||||
# Hack to get the signature of the class even though it subclasses | # Hack to get the signature of the class even though it subclasses | ||||
# Generic, which reimplements __new__. | # Generic, which reimplements __new__. | ||||
# See <https://bugs.python.org/issue40897> | # See <https://bugs.python.org/issue40897> | ||||
signature = inspect.signature(loader_cls.__init__) | signature = inspect.signature(loader_cls.__init__) | ||||
signature_str = str(signature).replace("self, ", "") | signature_str = str(signature).replace("self, ", "") | ||||
click.echo(f"Loader: {doc}\nsignature: {signature_str}") | click.echo(f"Loader: {doc}\nsignature: {signature_str}") |