Changeset View
Changeset View
Standalone View
Standalone View
swh/search/cli.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import functools | # WARNING: do not import unnecessary things here to keep cli startup time under | ||||
# control | |||||
import click | import click | ||||
from swh.core import config | |||||
from swh.core.cli import CONTEXT_SETTINGS | from swh.core.cli import CONTEXT_SETTINGS | ||||
from swh.journal.client import get_journal_client | |||||
from . import get_search | |||||
from .journal_client import process_journal_objects | |||||
from .api.server import load_and_check_config, app | |||||
@click.group(name="search", context_settings=CONTEXT_SETTINGS) | @click.group(name="search", context_settings=CONTEXT_SETTINGS) | ||||
@click.option( | @click.option( | ||||
"--config-file", | "--config-file", | ||||
"-C", | "-C", | ||||
default=None, | default=None, | ||||
type=click.Path(exists=True, dir_okay=False,), | type=click.Path(exists=True, dir_okay=False,), | ||||
help="Configuration file.", | help="Configuration file.", | ||||
) | ) | ||||
@click.pass_context | @click.pass_context | ||||
def cli(ctx, config_file): | def cli(ctx, config_file): | ||||
"""Software Heritage Search tools.""" | """Software Heritage Search tools.""" | ||||
ctx.ensure_object(dict) | from swh.core import config | ||||
ctx.ensure_object(dict) | |||||
conf = config.read(config_file) | conf = config.read(config_file) | ||||
ctx.obj["config"] = conf | ctx.obj["config"] = conf | ||||
@cli.command("initialize") | @cli.command("initialize") | ||||
@click.pass_context | @click.pass_context | ||||
def initialize(ctx): | def initialize(ctx): | ||||
"""Creates Elasticsearch indices.""" | """Creates Elasticsearch indices.""" | ||||
from . import get_search | |||||
search = get_search(**ctx.obj["config"]["search"]) | search = get_search(**ctx.obj["config"]["search"]) | ||||
search.initialize() | search.initialize() | ||||
print("Done.") | print("Done.") | ||||
@cli.group("journal-client") | @cli.group("journal-client") | ||||
@click.pass_context | @click.pass_context | ||||
def journal_client(ctx): | def journal_client(ctx): | ||||
Show All 9 Lines | @click.option( | ||||
type=int, | type=int, | ||||
help="Maximum number of objects to replay. Default is to run forever.", | help="Maximum number of objects to replay. Default is to run forever.", | ||||
) | ) | ||||
@click.pass_context | @click.pass_context | ||||
def journal_client_objects(ctx, stop_after_objects): | def journal_client_objects(ctx, stop_after_objects): | ||||
"""Listens for new objects from the SWH Journal, and schedules tasks | """Listens for new objects from the SWH Journal, and schedules tasks | ||||
to run relevant indexers (currently, only origin) | to run relevant indexers (currently, only origin) | ||||
on these new objects.""" | on these new objects.""" | ||||
import functools | |||||
from swh.journal.client import get_journal_client | |||||
from . import get_search | |||||
from .journal_client import process_journal_objects | |||||
config = ctx.obj["config"] | config = ctx.obj["config"] | ||||
journal_cfg = config["journal"] | journal_cfg = config["journal"] | ||||
client = get_journal_client( | client = get_journal_client( | ||||
cls="kafka", | cls="kafka", | ||||
object_types=["origin", "origin_visit"], | object_types=["origin", "origin_visit"], | ||||
stop_after_objects=stop_after_objects, | stop_after_objects=stop_after_objects, | ||||
**journal_cfg, | **journal_cfg, | ||||
Show All 19 Lines | |||||
@click.option("--port", default=5010, type=click.INT, help="Binding port of the server") | @click.option("--port", default=5010, type=click.INT, help="Binding port of the server") | ||||
@click.option( | @click.option( | ||||
"--debug/--nodebug", | "--debug/--nodebug", | ||||
default=True, | default=True, | ||||
help="Indicates if the server should run in debug mode", | help="Indicates if the server should run in debug mode", | ||||
) | ) | ||||
def rpc_server(config_path, host, port, debug): | def rpc_server(config_path, host, port, debug): | ||||
"""Starts a Software Heritage Indexer RPC HTTP server.""" | """Starts a Software Heritage Indexer RPC HTTP server.""" | ||||
from .api.server import load_and_check_config, app | |||||
api_cfg = load_and_check_config(config_path, type="any") | api_cfg = load_and_check_config(config_path, type="any") | ||||
app.config.update(api_cfg) | app.config.update(api_cfg) | ||||
app.run(host, port=int(port), debug=bool(debug)) | app.run(host, port=int(port), debug=bool(debug)) |