Changeset View
Changeset View
Standalone View
Standalone View
swh/indexer/cli.py
Show First 20 Lines • Show All 214 Lines • ▼ Show 20 Lines | |||||
@indexer_cli_group.command("journal-client") | @indexer_cli_group.command("journal-client") | ||||
@click.argument( | @click.argument( | ||||
"indexer", | "indexer", | ||||
type=click.Choice( | type=click.Choice( | ||||
[ | [ | ||||
"origin-intrinsic-metadata", | "origin-intrinsic-metadata", | ||||
"extrinsic-metadata", | "extrinsic-metadata", | ||||
"content-mimetype", | "content-mimetype", | ||||
"content-fossology-license", | |||||
"*", | "*", | ||||
] | ] | ||||
), | ), | ||||
required=False | required=False | ||||
# TODO: remove required=False after we stop using it | # TODO: remove required=False after we stop using it | ||||
) | ) | ||||
@click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") | @click.option("--scheduler-url", "-s", default=None, help="URL of the scheduler API") | ||||
@click.option( | @click.option( | ||||
▲ Show 20 Lines • Show All 100 Lines • ▼ Show 20 Lines | ): | ||||
if indexer in ("content-mimetype", "*"): | if indexer in ("content-mimetype", "*"): | ||||
from swh.indexer.mimetype import MimetypeIndexer | from swh.indexer.mimetype import MimetypeIndexer | ||||
object_types.add("content") | object_types.add("content") | ||||
idx = MimetypeIndexer() | idx = MimetypeIndexer() | ||||
idx.catch_exceptions = False # don't commit offsets if indexation failed | idx.catch_exceptions = False # don't commit offsets if indexation failed | ||||
worker_fns.append(idx.process_journal_objects) | worker_fns.append(idx.process_journal_objects) | ||||
if indexer in ("content-fossology-license", "*"): | |||||
from swh.indexer.fossology_license import FossologyLicenseIndexer | |||||
object_types.add("content") | |||||
idx = FossologyLicenseIndexer() | |||||
idx.catch_exceptions = False # don't commit offsets if indexation failed | |||||
worker_fns.append(idx.process_journal_objects) | |||||
if not worker_fns: | if not worker_fns: | ||||
raise click.ClickException(f"Unknown indexer: {indexer}") | raise click.ClickException(f"Unknown indexer: {indexer}") | ||||
client = get_journal_client( | client = get_journal_client( | ||||
cls="kafka", | cls="kafka", | ||||
brokers=brokers, | brokers=brokers, | ||||
prefix=prefix, | prefix=prefix, | ||||
group_id=group_id, | group_id=group_id, | ||||
▲ Show 20 Lines • Show All 42 Lines • Show Last 20 Lines |