Page MenuHomeSoftware Heritage
Differential D3913

cli: speedup the `swh` cli command startup time
ClosedPublic
Actions

Authored by douardda on Sep 10 2020, 4:14 PM.

Details

Reviewers
vlorentz
Group Reviewers
Reviewers
Maniphest Tasks
T2575: Investigate if/how we could improve `swh` cli command startup time
Commits
rDCIDX2ef7177bc2e2: cli: speedup the `swh` cli command startup time
Summary

by moving import statements in functions.

Related to T2575.

Diff Detail

Repository
rDCIDX Metadata indexer
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.

Event Timeline

douardda created this revision.Sep 10 2020, 4:14 PM
Herald added a reviewer: Reviewers. · View Herald TranscriptSep 10 2020, 4:14 PM
swh-public-ci added a comment.Sep 10 2020, 4:16 PM

Build has FAILED

Patch application report for D3913 (id=13772)

Rebasing onto c231873d56...

Current branch diff-target is up to date.
Changes applied before test
commit 939cb934e0794caf1278e3e63c4b013e0045f44d
Author: David Douard <david.douard@sdfa3.org>
Date:   Thu Sep 10 09:26:22 2020 +0200

    cli: speedup the `swh` cli command startup time
    
    by moving import statements in functions.
    
    Related to T2575.

Link to build: https://jenkins.softwareheritage.org/job/DCIDX/job/tests-on-diff/49/
See console output for more information: https://jenkins.softwareheritage.org/job/DCIDX/job/tests-on-diff/49/console

Harbormaster failed remote builds in B14999: Diff 13772!Sep 10 2020, 4:16 PM
vlorentz accepted this revision.Sep 10 2020, 4:38 PM
This revision is now accepted and ready to land.Sep 10 2020, 4:38 PM
douardda updated this revision to Diff 13839.Sep 11 2020, 3:08 PM

fix test_cli

mock needed being updated

swh-public-ci added a comment.Sep 11 2020, 3:09 PM

Build is green

Patch application report for D3913 (id=13839)

Rebasing onto c231873d56...

Current branch diff-target is up to date.
Changes applied before test
commit 2ef7177bc2e2916f32171269751485a230828355
Author: David Douard <david.douard@sdfa3.org>
Date:   Thu Sep 10 09:26:22 2020 +0200

    cli: speedup the `swh` cli command startup time
    
    by moving import statements in functions.
    
    Related to T2575.

See https://jenkins.softwareheritage.org/job/DCIDX/job/tests-on-diff/50/ for more details.

Harbormaster completed remote builds in B15064: Diff 13839.Sep 11 2020, 3:09 PM
Closed by commit rDCIDX2ef7177bc2e2: cli: speedup the `swh` cli command startup time (authored by douardda). · Explain WhySep 11 2020, 3:11 PM
This revision was automatically updated to reflect the committed changes.
douardda added a commit: rDCIDX2ef7177bc2e2: cli: speedup the `swh` cli command startup time.

Revision Contents
Changeset List

PathSize
swh/
indexer/
39 lines
tests/
2 lines

Diff 13840

View Options

swh/indexer/cli.py

# Copyright (C) 2019-2020 The Software Heritage developers # Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version # License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
import functools
import json
# WARNING: do not import unnecessary things here to keep cli startup time under
# control
import click import click
from swh.core import config
from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup
from swh.journal.client import get_journal_client
from swh.scheduler import get_scheduler
from swh.scheduler.cli_utils import schedule_origin_batches
from swh.storage import get_storage
from swh.indexer import metadata_dictionary
from swh.indexer.journal_client import process_journal_objects
from swh.indexer.storage import get_indexer_storage
from swh.indexer.storage.api.server import load_and_check_config, app
@click.group(name="indexer", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.group(name="indexer", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup)
@click.option( @click.option(
"--config-file", "--config-file",
"-C", "-C",
default=None, default=None,
type=click.Path(exists=True, dir_okay=False,), type=click.Path(exists=True, dir_okay=False,),
help="Configuration file.", help="Configuration file.",
) )
@click.pass_context @click.pass_context
def cli(ctx, config_file): def cli(ctx, config_file):
"""Software Heritage Indexer tools. """Software Heritage Indexer tools.
The Indexer is used to mine the content of the archive and extract derived The Indexer is used to mine the content of the archive and extract derived
information from archive source code artifacts. information from archive source code artifacts.
""" """
ctx.ensure_object(dict) from swh.core import config
ctx.ensure_object(dict)
conf = config.read(config_file) conf = config.read(config_file)
ctx.obj["config"] = conf ctx.obj["config"] = conf
def _get_api(getter, config, config_key, url): def _get_api(getter, config, config_key, url):
if url: if url:
config[config_key] = {"cls": "remote", "args": {"url": url}} config[config_key] = {"cls": "remote", "args": {"url": url}}
elif config_key not in config: elif config_key not in config:
raise click.ClickException("Missing configuration for {}".format(config_key)) raise click.ClickException("Missing configuration for {}".format(config_key))
return getter(**config[config_key]) return getter(**config[config_key])
@cli.group("mapping") @cli.group("mapping")
def mapping(): def mapping():
"""Manage Software Heritage Indexer mappings.""" """Manage Software Heritage Indexer mappings."""
pass pass
@mapping.command("list") @mapping.command("list")
def mapping_list(): def mapping_list():
"""Prints the list of known mappings.""" """Prints the list of known mappings."""
from swh.indexer import metadata_dictionary
mapping_names = [mapping.name for mapping in metadata_dictionary.MAPPINGS.values()] mapping_names = [mapping.name for mapping in metadata_dictionary.MAPPINGS.values()]
mapping_names.sort() mapping_names.sort()
for mapping_name in mapping_names: for mapping_name in mapping_names:
click.echo(mapping_name) click.echo(mapping_name)
@mapping.command("list-terms") @mapping.command("list-terms")
@click.option( @click.option(
"--exclude-mapping", multiple=True, help="Exclude the given mapping from the output" "--exclude-mapping", multiple=True, help="Exclude the given mapping from the output"
) )
@click.option( @click.option(
"--concise", "--concise",
is_flag=True, is_flag=True,
default=False, default=False,
help="Don't print the list of mappings supporting each term.", help="Don't print the list of mappings supporting each term.",
) )
def mapping_list_terms(concise, exclude_mapping): def mapping_list_terms(concise, exclude_mapping):
"""Prints the list of known CodeMeta terms, and which mappings """Prints the list of known CodeMeta terms, and which mappings
support them.""" support them."""
from swh.indexer import metadata_dictionary
properties = metadata_dictionary.list_terms() properties = metadata_dictionary.list_terms()
for (property_name, supported_mappings) in sorted(properties.items()): for (property_name, supported_mappings) in sorted(properties.items()):
supported_mappings = {m.name for m in supported_mappings} supported_mappings = {m.name for m in supported_mappings}
supported_mappings -= set(exclude_mapping) supported_mappings -= set(exclude_mapping)
if supported_mappings: if supported_mappings:
if concise: if concise:
click.echo(property_name) click.echo(property_name)
else: else:
click.echo("{}:".format(property_name)) click.echo("{}:".format(property_name))
click.echo("\t" + ", ".join(sorted(supported_mappings))) click.echo("\t" + ", ".join(sorted(supported_mappings)))
@mapping.command("translate") @mapping.command("translate")
@click.argument("mapping-name") @click.argument("mapping-name")
@click.argument("file", type=click.File("rb")) @click.argument("file", type=click.File("rb"))
def mapping_translate(mapping_name, file): def mapping_translate(mapping_name, file):
"""Prints the list of known mappings.""" """Prints the list of known mappings."""
import json
from swh.indexer import metadata_dictionary
mapping_cls = [ mapping_cls = [
cls for cls in metadata_dictionary.MAPPINGS.values() if cls.name == mapping_name cls for cls in metadata_dictionary.MAPPINGS.values() if cls.name == mapping_name
] ]
if not mapping_cls: if not mapping_cls:
raise click.ClickException("Unknown mapping {}".format(mapping_name)) raise click.ClickException("Unknown mapping {}".format(mapping_name))
assert len(mapping_cls) == 1 assert len(mapping_cls) == 1
mapping_cls = mapping_cls[0] mapping_cls = mapping_cls[0]
mapping = mapping_cls() mapping = mapping_cls()
Show All 15 Lines@click.option(
default=False, default=False,
help="List only what would be scheduled.", help="List only what would be scheduled.",
) )
@click.pass_context @click.pass_context
def schedule(ctx, scheduler_url, storage_url, indexer_storage_url, dry_run): def schedule(ctx, scheduler_url, storage_url, indexer_storage_url, dry_run):
"""Manipulate Software Heritage Indexer tasks. """Manipulate Software Heritage Indexer tasks.
Via SWH Scheduler's API.""" Via SWH Scheduler's API."""
from swh.scheduler import get_scheduler
from swh.storage import get_storage
from swh.indexer.storage import get_indexer_storage
ctx.obj["indexer_storage"] = _get_api( ctx.obj["indexer_storage"] = _get_api(
get_indexer_storage, ctx.obj["config"], "indexer_storage", indexer_storage_url get_indexer_storage, ctx.obj["config"], "indexer_storage", indexer_storage_url
) )
ctx.obj["storage"] = _get_api( ctx.obj["storage"] = _get_api(
get_storage, ctx.obj["config"], "storage", storage_url get_storage, ctx.obj["config"], "storage", storage_url
) )
ctx.obj["scheduler"] = _get_api( ctx.obj["scheduler"] = _get_api(
get_scheduler, ctx.obj["config"], "scheduler", scheduler_url get_scheduler, ctx.obj["config"], "scheduler", scheduler_url
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines@click.option(
show_default=True, show_default=True,
help="Name of the task type to schedule.", help="Name of the task type to schedule.",
) )
@click.pass_context @click.pass_context
def schedule_origin_metadata_reindex( def schedule_origin_metadata_reindex(
ctx, origin_batch_size, tool_ids, mappings, task_type ctx, origin_batch_size, tool_ids, mappings, task_type
): ):
"""Schedules indexing tasks for origins that were already indexed.""" """Schedules indexing tasks for origins that were already indexed."""
from swh.scheduler.cli_utils import schedule_origin_batches
idx_storage = ctx.obj["indexer_storage"] idx_storage = ctx.obj["indexer_storage"]
scheduler = ctx.obj["scheduler"] scheduler = ctx.obj["scheduler"]
origins = list_origins_by_producer(idx_storage, mappings, tool_ids) origins = list_origins_by_producer(idx_storage, mappings, tool_ids)
kwargs = {"policy_update": "update-dups"} kwargs = {"policy_update": "update-dups"}
schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs) schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs)
Show All 27 Linesdef journal_client(
brokers, brokers,
prefix, prefix,
group_id, group_id,
stop_after_objects, stop_after_objects,
): ):
"""Listens for new objects from the SWH Journal, and schedules tasks """Listens for new objects from the SWH Journal, and schedules tasks
to run relevant indexers (currently, only origin-intrinsic-metadata) to run relevant indexers (currently, only origin-intrinsic-metadata)
on these new objects.""" on these new objects."""
import functools
from swh.journal.client import get_journal_client
from swh.scheduler import get_scheduler
from swh.indexer.journal_client import process_journal_objects
scheduler = _get_api(get_scheduler, ctx.obj["config"], "scheduler", scheduler_url) scheduler = _get_api(get_scheduler, ctx.obj["config"], "scheduler", scheduler_url)
client = get_journal_client( client = get_journal_client(
cls="kafka", cls="kafka",
brokers=brokers, brokers=brokers,
prefix=prefix, prefix=prefix,
group_id=group_id, group_id=group_id,
object_types=["origin_visit"], object_types=["origin_visit"],
Show All 21 Lines
@click.option("--port", default=5007, type=click.INT, help="Binding port of the server") @click.option("--port", default=5007, type=click.INT, help="Binding port of the server")
@click.option( @click.option(
"--debug/--nodebug", "--debug/--nodebug",
default=True, default=True,
help="Indicates if the server should run in debug mode", help="Indicates if the server should run in debug mode",
) )
def rpc_server(config_path, host, port, debug): def rpc_server(config_path, host, port, debug):
"""Starts a Software Heritage Indexer RPC HTTP server.""" """Starts a Software Heritage Indexer RPC HTTP server."""
from swh.indexer.storage.api.server import load_and_check_config, app
api_cfg = load_and_check_config(config_path, type="any") api_cfg = load_and_check_config(config_path, type="any")
app.config.update(api_cfg) app.config.update(api_cfg)
app.run(host, port=int(port), debug=bool(debug)) app.run(host, port=int(port), debug=bool(debug))
def main(): def main():
return cli(auto_envvar_prefix="SWH_INDEXER") return cli(auto_envvar_prefix="SWH_INDEXER")
if __name__ == "__main__": if __name__ == "__main__":
main() main()
View Options

swh/indexer/tests/test_cli.py

Show First 20 Lines • Show All 76 Lines • ▼ Show 20 Linesdef _assert_tasks_for_origins(tasks, origins):
for task in tasks: for task in tasks:
assert task["arguments"]["kwargs"] == expected_kwargs, task assert task["arguments"]["kwargs"] == expected_kwargs, task
assert _origins_in_task_args(tasks) == set(["file://dev/%04d" % i for i in origins]) assert _origins_in_task_args(tasks) == set(["file://dev/%04d" % i for i in origins])
def invoke(scheduler, catch_exceptions, args): def invoke(scheduler, catch_exceptions, args):
runner = CliRunner() runner = CliRunner()
with patch( with patch(
"swh.indexer.cli.get_scheduler" "swh.scheduler.get_scheduler"
) as get_scheduler_mock, tempfile.NamedTemporaryFile( ) as get_scheduler_mock, tempfile.NamedTemporaryFile(
"a", suffix=".yml" "a", suffix=".yml"
) as config_fd: ) as config_fd:
config_fd.write(CLI_CONFIG) config_fd.write(CLI_CONFIG)
config_fd.seek(0) config_fd.seek(0)
get_scheduler_mock.return_value = scheduler get_scheduler_mock.return_value = scheduler
result = runner.invoke(cli, ["-C" + config_fd.name] + args) result = runner.invoke(cli, ["-C" + config_fd.name] + args)
if not catch_exceptions and result.exception: if not catch_exceptions and result.exception:
▲ Show 20 Lines • Show All 289 Lines • Show Last 20 Lines
About · Developer information · Legal