Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/cli/task.py
| # Copyright (C) 2016-2020 The Software Heritage developers | # Copyright (C) 2016-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import datetime | import datetime | ||||
| import json | import json | ||||
| import itertools | import itertools | ||||
| import locale | import locale | ||||
| import logging | import logging | ||||
| import arrow | import arrow | ||||
| import csv | import csv | ||||
| import click | import click | ||||
| from typing import Any, Dict | from typing import Any, Dict | ||||
| from itertools import islice | |||||
| from . import cli | from . import cli | ||||
| locale.setlocale(locale.LC_ALL, "") | locale.setlocale(locale.LC_ALL, "") | ||||
| ARROW_LOCALE = locale.getlocale(locale.LC_TIME)[0] | ARROW_LOCALE = locale.getlocale(locale.LC_TIME)[0] | ||||
| ▲ Show 20 Lines • Show All 268 Lines • ▼ Show 20 Lines | @click.option( | ||||
| "-b", | "-b", | ||||
| "origin_batch_size", | "origin_batch_size", | ||||
| default=10, | default=10, | ||||
| show_default=True, | show_default=True, | ||||
| type=int, | type=int, | ||||
| help="Number of origins per task", | help="Number of origins per task", | ||||
| ) | ) | ||||
| @click.option( | @click.option( | ||||
| "--min-id", | "--page-token", | ||||
| default=0, | default=0, | ||||
| show_default=True, | show_default=True, | ||||
| type=int, | type=str, | ||||
| help="Only schedule tasks for origins whose ID is greater", | help="Only schedule tasks for origins whose ID is greater", | ||||
| ) | ) | ||||
| @click.option( | @click.option( | ||||
| "--max-id", | "--limit", | ||||
| default=None, | default=None, | ||||
| type=int, | type=int, | ||||
| help="Only schedule tasks for origins whose ID is lower", | help="Limit the tasks scheduling up to this number of tasks", | ||||
| ) | ) | ||||
| @click.option("--storage-url", "-g", help="URL of the (graph) storage API") | @click.option("--storage-url", "-g", help="URL of the (graph) storage API") | ||||
| @click.option( | @click.option( | ||||
| "--dry-run/--no-dry-run", | "--dry-run/--no-dry-run", | ||||
| is_flag=True, | is_flag=True, | ||||
| default=False, | default=False, | ||||
| help="List only what would be scheduled.", | help="List only what would be scheduled.", | ||||
| ) | ) | ||||
| @click.pass_context | @click.pass_context | ||||
| def schedule_origin_metadata_index( | def schedule_origin_metadata_index( | ||||
| ctx, type, options, storage_url, origin_batch_size, min_id, max_id, dry_run | ctx, type, options, storage_url, origin_batch_size, page_token, limit, dry_run | ||||
| ): | ): | ||||
| """Schedules tasks for origins that are already known. | """Schedules tasks for origins that are already known. | ||||
| The first argument is the name of the task type, further ones are | The first argument is the name of the task type, further ones are | ||||
| keyword argument(s) of the task in the form key=value, where value is | keyword argument(s) of the task in the form key=value, where value is | ||||
| in YAML format. | in YAML format. | ||||
| Usage sample: | Usage sample: | ||||
| Show All 9 Lines | ): | ||||
| storage = get_storage("remote", url=storage_url) | storage = get_storage("remote", url=storage_url) | ||||
| if dry_run: | if dry_run: | ||||
| scheduler = None | scheduler = None | ||||
| (args, kw) = parse_options(options) | (args, kw) = parse_options(options) | ||||
| if args: | if args: | ||||
| raise click.ClickException("Only keywords arguments are allowed.") | raise click.ClickException("Only keywords arguments are allowed.") | ||||
| origins = iter_origins(storage, origin_from=min_id, origin_to=max_id) | origins = iter_origins(storage, page_token=page_token) | ||||
| origin_urls = (origin.url for origin in origins) | if limit: | ||||
| origins = islice(origins, limit) | |||||
| origin_urls = (origin.url for origin in origins) | |||||
| schedule_origin_batches(scheduler, type, origin_urls, origin_batch_size, kw) | schedule_origin_batches(scheduler, type, origin_urls, origin_batch_size, kw) | ||||
| @task.command("list-pending") | @task.command("list-pending") | ||||
| @click.argument("task-types", required=True, nargs=-1) | @click.argument("task-types", required=True, nargs=-1) | ||||
| @click.option( | @click.option( | ||||
| "--limit", | "--limit", | ||||
| "-l", | "-l", | ||||
| ▲ Show 20 Lines • Show All 362 Lines • Show Last 20 Lines | |||||