Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/cli.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2021 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
# WARNING: do not import unnecessary things here to keep cli startup time under | # WARNING: do not import unnecessary things here to keep cli startup time under | ||||
# control | # control | ||||
import logging | import logging | ||||
from typing import Any | from typing import Any | ||||
Show All 33 Lines | if name not in LOADERS: | ||||
"Invalid loader %s: only supported loaders are %s" | "Invalid loader %s: only supported loaders are %s" | ||||
% (name, SUPPORTED_LOADERS) | % (name, SUPPORTED_LOADERS) | ||||
) | ) | ||||
registry_entry = LOADERS[name].load()() | registry_entry = LOADERS[name].load()() | ||||
logger.debug(f"registry: {registry_entry}") | logger.debug(f"registry: {registry_entry}") | ||||
loader_cls = registry_entry["loader"] | loader_cls = registry_entry["loader"] | ||||
logger.debug(f"loader class: {loader_cls}") | logger.debug(f"loader class: {loader_cls}") | ||||
return loader_cls(**kwargs) | return loader_cls.from_config(**kwargs) | ||||
@swh_cli_group.group(name="loader", context_settings=CONTEXT_SETTINGS) | @swh_cli_group.group(name="loader", context_settings=CONTEXT_SETTINGS) | ||||
@click.option( | |||||
"--config-file", | |||||
"-C", | |||||
default=None, | |||||
type=click.Path(exists=True, dir_okay=False,), | |||||
help="Configuration file.", | |||||
) | |||||
@click.pass_context | @click.pass_context | ||||
def loader(ctx): | def loader(ctx, config_file): | ||||
"""Loader cli tools | """Loader cli tools | ||||
""" | """ | ||||
pass | from os import environ | ||||
from swh.core.config import read | |||||
ctx.ensure_object(dict) | |||||
logger.debug("ctx: %s", ctx) | |||||
if not config_file: | |||||
config_file = environ.get("SWH_CONFIG_FILENAME") | |||||
ctx.obj["config"] = read(config_file) | |||||
logger.debug("config_file: %s", config_file) | |||||
logger.debug("config: ", ctx.obj["config"]) | |||||
@loader.command(name="run", context_settings=CONTEXT_SETTINGS) | @loader.command(name="run", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | @click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | ||||
@click.argument("url") | @click.argument("url") | ||||
@click.argument("options", nargs=-1) | @click.argument("options", nargs=-1) | ||||
@click.pass_context | @click.pass_context | ||||
def run(ctx, type, url, options): | def run(ctx, type, url, options): | ||||
"""Ingest with loader <type> the origin located at <url>""" | """Ingest with loader <type> the origin located at <url>""" | ||||
import iso8601 | import iso8601 | ||||
from swh.scheduler.cli.utils import parse_options | from swh.scheduler.cli.utils import parse_options | ||||
conf = ctx.obj.get("config", {}) | |||||
if "storage" not in conf: | |||||
raise ValueError("Missing storage configuration key") | |||||
(_, kw) = parse_options(options) | (_, kw) = parse_options(options) | ||||
logger.debug(f"kw: {kw}") | logger.debug(f"kw: {kw}") | ||||
visit_date = kw.get("visit_date") | visit_date = kw.get("visit_date") | ||||
if visit_date and isinstance(visit_date, str): | if visit_date and isinstance(visit_date, str): | ||||
visit_date = iso8601.parse_date(visit_date) | visit_date = iso8601.parse_date(visit_date) | ||||
kw["visit_date"] = visit_date | kw["visit_date"] = visit_date | ||||
loader = get_loader(type, url=url, **kw) | loader = get_loader(type, url=url, storage=conf["storage"], **kw) | ||||
result = loader.load() | result = loader.load() | ||||
click.echo(result) | click.echo(result) | ||||
@loader.command(name="list", context_settings=CONTEXT_SETTINGS) | @loader.command(name="list", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("type", default="all", type=click.Choice(["all"] + SUPPORTED_LOADERS)) | @click.argument("type", default="all", type=click.Choice(["all"] + SUPPORTED_LOADERS)) | ||||
@click.pass_context | @click.pass_context | ||||
def list(ctx, type): | def list(ctx, type): | ||||
Show All 18 Lines |