Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/cli.py
Show First 20 Lines • Show All 209 Lines • ▼ Show 20 Lines | ): | ||||
kwargs = {"retries_left": 1} | kwargs = {"retries_left": 1} | ||||
schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs) | schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs) | ||||
@indexer_cli_group.command("journal-client") | @indexer_cli_group.command("journal-client") | ||||
@click.argument( | @click.argument( | ||||
"indexer", | "indexer", | ||||
type=click.Choice(["origin-intrinsic-metadata", "extrinsic-metadata", "*"]), | type=click.Choice( | ||||
[ | |||||
"origin-intrinsic-metadata", | |||||
"extrinsic-metadata", | |||||
"content-mimetype", | |||||
"*", | |||||
] | |||||
), | |||||
vlorentz: you'll need to update that | |||||
Done Inline Actionsyes, thx, it's done locally. ardumont: yes, thx, it's done locally.
ongoing tests showed it to me ;) | |||||
required=False | required=False | ||||
# TODO: remove required=False after we stop using it | # TODO: remove required=False after we stop using it | ||||
) | ) | ||||
@click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") | @click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") | ||||
@click.option( | @click.option( | ||||
"--origin-metadata-task-type", | "--origin-metadata-task-type", | ||||
default="index-origin-metadata", | default="index-origin-metadata", | ||||
help="Name of the task running the origin metadata indexer.", | help="Name of the task running the origin metadata indexer.", | ||||
▲ Show 20 Lines • Show All 89 Lines • ▼ Show 20 Lines | ): | ||||
if indexer in ("extrinsic-metadata", "*"): | if indexer in ("extrinsic-metadata", "*"): | ||||
from swh.indexer.metadata import ExtrinsicMetadataIndexer | from swh.indexer.metadata import ExtrinsicMetadataIndexer | ||||
object_types.add("raw_extrinsic_metadata") | object_types.add("raw_extrinsic_metadata") | ||||
idx = ExtrinsicMetadataIndexer() | idx = ExtrinsicMetadataIndexer() | ||||
idx.catch_exceptions = False # don't commit offsets if indexation failed | idx.catch_exceptions = False # don't commit offsets if indexation failed | ||||
worker_fns.append(idx.process_journal_objects) | worker_fns.append(idx.process_journal_objects) | ||||
if indexer in ("content-mimetype", "*"): | |||||
from swh.indexer.mimetype import MimetypeIndexer | |||||
object_types.add("content") | |||||
idx = MimetypeIndexer() | |||||
idx.catch_exceptions = False # don't commit offsets if indexation failed | |||||
worker_fns.append(idx.process_journal_objects) | |||||
if not worker_fns: | if not worker_fns: | ||||
raise click.ClickException(f"Unknown indexer: {indexer}") | raise click.ClickException(f"Unknown indexer: {indexer}") | ||||
client = get_journal_client( | client = get_journal_client( | ||||
cls="kafka", | cls="kafka", | ||||
brokers=brokers, | brokers=brokers, | ||||
prefix=prefix, | prefix=prefix, | ||||
group_id=group_id, | group_id=group_id, | ||||
▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines |
you'll need to update that