Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/cli.py
# Copyright (C) 2019-2021 The Software Heritage developers | # Copyright (C) 2019-2022 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
# WARNING: do not import unnecessary things here to keep cli startup time under | # WARNING: do not import unnecessary things here to keep cli startup time under | ||||
# control | # control | ||||
import logging | import logging | ||||
from typing import Any | from typing import Any | ||||
▲ Show 20 Lines • Show All 62 Lines • ▼ Show 20 Lines | def loader(ctx, config_file): | ||||
ctx.ensure_object(dict) | ctx.ensure_object(dict) | ||||
logger.debug("ctx: %s", ctx) | logger.debug("ctx: %s", ctx) | ||||
if not config_file: | if not config_file: | ||||
config_file = environ.get("SWH_CONFIG_FILENAME") | config_file = environ.get("SWH_CONFIG_FILENAME") | ||||
ctx.obj["config"] = read(config_file) | ctx.obj["config"] = read(config_file) | ||||
logger.debug("config_file: %s", config_file) | logger.debug("config_file: %s", config_file) | ||||
logger.debug("config: ", ctx.obj["config"]) | logger.debug("config: %s", ctx.obj["config"]) | ||||
@loader.command(name="run", context_settings=CONTEXT_SETTINGS) | @loader.command(name="run", context_settings=CONTEXT_SETTINGS) | ||||
@click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | @click.argument("type", type=click.Choice(SUPPORTED_LOADERS)) | ||||
@click.argument("url") | @click.argument("url") | ||||
@click.argument("options", nargs=-1) | @click.argument("options", nargs=-1) | ||||
@click.pass_context | @click.pass_context | ||||
def run(ctx, type, url, options): | def run(ctx, type, url, options): | ||||
"""Ingest with loader <type> the origin located at <url>""" | """Ingest with loader <type> the origin located at <url>""" | ||||
import iso8601 | import iso8601 | ||||
from swh.scheduler.cli.utils import parse_options | from swh.scheduler.cli.utils import parse_options | ||||
conf = ctx.obj.get("config", {}) | conf = ctx.obj.get("config", {}) | ||||
if "storage" not in conf: | if "storage" not in conf: | ||||
raise ValueError("Missing storage configuration key") | logger.warning( | ||||
"No storage configuration detected, using an in-memory storage instead." | |||||
) | |||||
conf["storage"] = {"cls": "memory"} | |||||
vlorentz: could you make it a warning? | |||||
(_, kw) = parse_options(options) | (_, kw) = parse_options(options) | ||||
logger.debug(f"kw: {kw}") | logger.debug(f"kw: {kw}") | ||||
visit_date = kw.get("visit_date") | visit_date = kw.get("visit_date") | ||||
if visit_date and isinstance(visit_date, str): | if visit_date and isinstance(visit_date, str): | ||||
visit_date = iso8601.parse_date(visit_date) | visit_date = iso8601.parse_date(visit_date) | ||||
kw["visit_date"] = visit_date | kw["visit_date"] = visit_date | ||||
loader = get_loader( | loader = get_loader( | ||||
Show All 36 Lines |
could you make it a warning?