No OneTemporary
Actions

Size

44 KB

Subscribers

None

View Options

	diff --git a/swh/scheduler/cli/__init__.py b/swh/scheduler/cli/__init__.py
	index 57e26a4..4cd81c6 100644
	--- a/swh/scheduler/cli/__init__.py
	+++ b/swh/scheduler/cli/__init__.py
	@@ -1,91 +1,93 @@
	# Copyright (C) 2016-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control
	import logging

	import click

	from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup


	@click.group(name="scheduler", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup)
	@click.option(
	"--config-file",
	"-C",
	default=None,
	type=click.Path(exists=True, dir_okay=False,),
	help="Configuration file.",
	)
	@click.option(
	"--database",
	"-d",
	default=None,
	help="Scheduling database DSN (imply cls is 'local')",
	)
	@click.option(
	"--url", "-u", default=None, help="Scheduler's url access (imply cls is 'remote')"
	)
	@click.option(
	"--no-stdout", is_flag=True, default=False, help="Do NOT output logs on the console"
	)
	@click.pass_context
	def cli(ctx, config_file, database, url, no_stdout):
	"""Software Heritage Scheduler tools.

	Use a local scheduler instance by default (plugged to the
	main scheduler db).
	"""
	try:
	from psycopg2 import OperationalError
	except ImportError:

	class OperationalError(Exception):
	pass

	from swh.core import config
	from swh.scheduler import get_scheduler, DEFAULT_CONFIG

	ctx.ensure_object(dict)

	logger = logging.getLogger(__name__)
	scheduler = None
	conf = config.read(config_file, DEFAULT_CONFIG)
	if "scheduler" not in conf:
	raise ValueError("missing 'scheduler' configuration")

	if database:
	conf["scheduler"]["cls"] = "local"
	conf["scheduler"]["args"]["db"] = database
	elif url:
	conf["scheduler"]["cls"] = "remote"
	conf["scheduler"]["args"] = {"url": url}
	sched_conf = conf["scheduler"]
	try:
	logger.debug("Instantiating scheduler with %s" % (sched_conf))
	scheduler = get_scheduler(**sched_conf)
	except (ValueError, OperationalError):
	# it's the subcommand to decide whether not having a proper
	# scheduler instance is a problem.
	pass

	ctx.obj["scheduler"] = scheduler
	ctx.obj["config"] = conf


	from . import admin, celery_monitor, task, task_type # noqa


	def main():
	import click.core

	click.core.DEPRECATED_HELP_NOTICE = """

	DEPRECATED! Please use the command 'swh scheduler'."""
	cli.deprecated = True
	return cli(auto_envvar_prefix="SWH_SCHEDULER")


	if __name__ == "__main__":
	main()
	diff --git a/swh/scheduler/cli/admin.py b/swh/scheduler/cli/admin.py
	index a864897..9d8fdd2 100644
	--- a/swh/scheduler/cli/admin.py
	+++ b/swh/scheduler/cli/admin.py
	@@ -1,108 +1,110 @@
	# Copyright (C) 2016-2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control
	import logging
	import time

	import click

	from . import cli


	@cli.command("start-runner")
	@click.option(
	"--period",
	"-p",
	default=0,
	help=(
	"Period (in s) at witch pending tasks are checked and "
	"executed. Set to 0 (default) for a one shot."
	),
	)
	@click.pass_context
	def runner(ctx, period):
	"""Starts a swh-scheduler runner service.

	This process is responsible for checking for ready-to-run tasks and
	schedule them."""
	from swh.scheduler.celery_backend.runner import run_ready_tasks
	from swh.scheduler.celery_backend.config import build_app

	app = build_app(ctx.obj["config"].get("celery"))
	app.set_current()

	logger = logging.getLogger(__name__ + ".runner")
	scheduler = ctx.obj["scheduler"]
	logger.debug("Scheduler %s" % scheduler)
	try:
	while True:
	logger.debug("Run ready tasks")
	try:
	ntasks = len(run_ready_tasks(scheduler, app))
	if ntasks:
	logger.info("Scheduled %s tasks", ntasks)
	except Exception:
	logger.exception("Unexpected error in run_ready_tasks()")
	if not period:
	break
	time.sleep(period)
	except KeyboardInterrupt:
	ctx.exit(0)


	@cli.command("start-listener")
	@click.pass_context
	def listener(ctx):
	"""Starts a swh-scheduler listener service.

	This service is responsible for listening at task lifecycle events and
	handle their workflow status in the database."""
	scheduler_backend = ctx.obj["scheduler"]
	if not scheduler_backend:
	raise ValueError("Scheduler class (local/remote) must be instantiated")

	broker = (
	ctx.obj["config"]
	.get("celery", {})
	.get("task_broker", "amqp://guest@localhost/%2f")
	)

	from swh.scheduler.celery_backend.pika_listener import get_listener

	listener = get_listener(broker, "celeryev.listener", scheduler_backend)
	try:
	listener.start_consuming()
	finally:
	listener.stop_consuming()


	@cli.command("rpc-serve")
	@click.option("--host", default="0.0.0.0", help="Host to run the scheduler server api")
	@click.option("--port", default=5008, type=click.INT, help="Binding port of the server")
	@click.option(
	"--debug/--nodebug",
	default=None,
	help=(
	"Indicates if the server should run in debug mode. "
	"Defaults to True if log-level is DEBUG, False otherwise."
	),
	)
	@click.pass_context
	def rpc_server(ctx, host, port, debug):
	"""Starts a swh-scheduler API HTTP server.
	"""
	if ctx.obj["config"]["scheduler"]["cls"] == "remote":
	click.echo(
	"The API server can only be started with a 'local' " "configuration",
	err=True,
	)
	ctx.exit(1)

	from swh.scheduler.api import server

	server.app.config.update(ctx.obj["config"])
	if debug is None:
	debug = ctx.obj["log_level"] <= logging.DEBUG
	server.app.run(host, port=port, debug=bool(debug))
	diff --git a/swh/scheduler/cli/celery_monitor.py b/swh/scheduler/cli/celery_monitor.py
	index 7204ead..cf80d0e 100644
	--- a/swh/scheduler/cli/celery_monitor.py
	+++ b/swh/scheduler/cli/celery_monitor.py
	@@ -1,157 +1,160 @@
	# Copyright (C) 2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-from ast import literal_eval
	-import csv
	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control
	import logging
	import sys
	import time
	from typing import Any, Dict, Optional

	import click

	from . import cli

	logger = logging.getLogger(__name__)


	def destination_from_pattern(ctx: click.Context, pattern: Optional[str]):
	"""Get the celery destination pattern from host and type values"""
	if pattern is None:
	logger.debug("Matching all workers")
	elif "*" in pattern:
	ctx.obj["inspect"].pattern = pattern
	ctx.obj["inspect"].matcher = "glob"
	logger.debug("Using glob pattern %s", pattern)
	else:
	destination = pattern.split(",")
	ctx.obj["inspect"].destination = destination
	logger.debug("Using destinations %s", ", ".join(destination))


	@cli.group("celery-monitor")
	@click.option(
	"--timeout", type=float, default=3.0, help="Timeout for celery remote control"
	)
	@click.option("--pattern", help="Celery destination pattern", default=None)
	@click.pass_context
	def celery_monitor(ctx: click.Context, timeout: float, pattern: Optional[str]) -> None:
	"""Monitoring of Celery"""
	from swh.scheduler.celery_backend.config import app

	ctx.obj["timeout"] = timeout
	ctx.obj["inspect"] = app.control.inspect(timeout=timeout)

	destination_from_pattern(ctx, pattern)


	@celery_monitor.command("ping-workers")
	@click.pass_context
	def ping_workers(ctx: click.Context) -> None:
	"""Check which workers respond to the celery remote control"""

	response_times = {}

	def ping_callback(response):
	rtt = time.monotonic() - ping_time
	for destination in response:
	logger.debug("Got ping response from %s: %r", destination, response)
	response_times[destination] = rtt

	ctx.obj["inspect"].callback = ping_callback

	ping_time = time.monotonic()
	ret = ctx.obj["inspect"].ping()

	if not ret:
	logger.info("No response in %f seconds", time.monotonic() - ping_time)
	ctx.exit(1)

	for destination in ret:
	logger.info(
	"Got response from %s in %f seconds",
	destination,
	response_times[destination],
	)

	ctx.exit(0)


	@celery_monitor.command("list-running")
	@click.option(
	"--format",
	help="Output format",
	default="pretty",
	type=click.Choice(["pretty", "csv"]),
	)
	@click.pass_context
	def list_running(ctx: click.Context, format: str):
	"""List running tasks on the lister workers"""
	+ from ast import literal_eval
	+ import csv
	+
	response_times = {}

	def active_callback(response):
	rtt = time.monotonic() - active_time
	for destination in response:
	response_times[destination] = rtt

	ctx.obj["inspect"].callback = active_callback

	active_time = time.monotonic()
	ret = ctx.obj["inspect"].active()

	if not ret:
	logger.info("No response in %f seconds", time.monotonic() - active_time)
	ctx.exit(1)

	def pretty_task_arguments(task: Dict[str, Any]) -> str:
	arg_list = []
	for arg in task["args"]:
	arg_list.append(repr(arg))
	for k, v in task["kwargs"].items():
	arg_list.append(f"{k}={v!r}")

	return f'{task["name"]}({", ".join(arg_list)})'

	def get_task_data(worker: str, task: Dict[str, Any]) -> Dict[str, Any]:
	duration = time.time() - task["time_start"]
	return {
	"worker": worker,
	"name": task["name"],
	"args": literal_eval(task["args"]),
	"kwargs": literal_eval(task["kwargs"]),
	"duration": duration,
	"worker_pid": task["worker_pid"],
	}

	if format == "csv":
	writer = csv.DictWriter(
	sys.stdout, ["worker", "name", "args", "kwargs", "duration", "worker_pid"]
	)
	writer.writeheader()

	def output(data: Dict[str, Any]):
	writer.writerow(data)

	elif format == "pretty":

	def output(data: Dict[str, Any]):
	print(
	f"{data['worker']}: {pretty_task_arguments(data)} "
	f"[for {data['duration']:f}s, pid={data['worker_pid']}]"
	)

	else:
	logger.error("Unknown format %s", format)
	ctx.exit(127)

	for worker, active in sorted(ret.items()):
	if not active:
	logger.info("%s: no active tasks", worker)
	continue

	for task in sorted(active, key=lambda t: t["time_start"]):
	output(get_task_data(worker, task))

	ctx.exit(0)
	diff --git a/swh/scheduler/cli/task.py b/swh/scheduler/cli/task.py
	index 4b1d8aa..e415a64 100644
	--- a/swh/scheduler/cli/task.py
	+++ b/swh/scheduler/cli/task.py
	@@ -1,746 +1,763 @@
	# Copyright (C) 2016-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-import datetime
	-import json
	-import itertools
	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control
	import locale
	import logging

	-import arrow
	-import csv
	import click

	-from typing import Any, Dict, Optional, Iterator
	-from itertools import islice
	-
	-from swh.model.model import Origin
	-from swh.storage.interface import StorageInterface
	+from typing import Any, Dict, Optional, Iterator, TYPE_CHECKING

	from . import cli

	+if TYPE_CHECKING:
	+ # importing swh.storage.interface triggers the load of 300+ modules, so...
	+ from swh.model.model import Origin
	+ from swh.storage.interface import StorageInterface
	+

	locale.setlocale(locale.LC_ALL, "")
	ARROW_LOCALE = locale.getlocale(locale.LC_TIME)[0]


	class DateTimeType(click.ParamType):
	name = "time and date"

	def convert(self, value, param, ctx):
	+ import arrow
	+
	if not isinstance(value, arrow.Arrow):
	value = arrow.get(value)

	return value


	DATETIME = DateTimeType()
	CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])


	def format_dict(d):
	+ import datetime
	+ import arrow
	+
	ret = {}
	for k, v in d.items():
	if isinstance(v, (arrow.Arrow, datetime.date, datetime.datetime)):
	v = arrow.get(v).format()
	elif isinstance(v, dict):
	v = format_dict(v)
	ret[k] = v
	return ret


	def pretty_print_list(list, indent=0):
	"""Pretty-print a list"""
	return "".join("%s%r\n" % (" " * indent, item) for item in list)


	def pretty_print_dict(dict, indent=0):
	"""Pretty-print a list"""
	return "".join(
	"%s%s: %r\n" % (" " * indent, click.style(key, bold=True), value)
	for key, value in sorted(dict.items())
	)


	def pretty_print_run(run, indent=4):
	fmt = (
	"{indent}{backend_id} [{status}]\n"
	"{indent} scheduled: {scheduled} [{started}:{ended}]"
	)
	return fmt.format(indent=" " * indent, **format_dict(run))


	def pretty_print_task(task, full=False):
	"""Pretty-print a task

	If 'full' is True, also print the status and priority fields.

	+ >>> import datetime
	>>> task = {
	... 'id': 1234,
	... 'arguments': {
	... 'args': ['foo', 'bar', True],
	... 'kwargs': {'key': 'value', 'key2': 42},
	... },
	... 'current_interval': datetime.timedelta(hours=1),
	... 'next_run': datetime.datetime(2019, 2, 21, 13, 52, 35, 407818),
	... 'policy': 'oneshot',
	... 'priority': None,
	... 'status': 'next_run_not_scheduled',
	... 'type': 'test_task',
	... }
	>>> print(click.unstyle(pretty_print_task(task)))
	Task 1234
	Next run: ... (2019-02-21 13:52:35+00:00)
	Interval: 1:00:00
	Type: test_task
	Policy: oneshot
	Args:
	'foo'
	'bar'
	True
	Keyword args:
	key: 'value'
	key2: 42
	<BLANKLINE>
	>>> print(click.unstyle(pretty_print_task(task, full=True)))
	Task 1234
	Next run: ... (2019-02-21 13:52:35+00:00)
	Interval: 1:00:00
	Type: test_task
	Policy: oneshot
	Status: next_run_not_scheduled
	Priority:\x20
	Args:
	'foo'
	'bar'
	True
	Keyword args:
	key: 'value'
	key2: 42
	<BLANKLINE>
	"""
	+ import arrow
	+
	next_run = arrow.get(task["next_run"])
	lines = [
	"%s %s\n" % (click.style("Task", bold=True), task["id"]),
	click.style(" Next run: ", bold=True),
	"%s (%s)" % (next_run.humanize(locale=ARROW_LOCALE), next_run.format()),
	"\n",
	click.style(" Interval: ", bold=True),
	str(task["current_interval"]),
	"\n",
	click.style(" Type: ", bold=True),
	task["type"] or "",
	"\n",
	click.style(" Policy: ", bold=True),
	task["policy"] or "",
	"\n",
	]
	if full:
	lines += [
	click.style(" Status: ", bold=True),
	task["status"] or "",
	"\n",
	click.style(" Priority: ", bold=True),
	task["priority"] or "",
	"\n",
	]
	lines += [
	click.style(" Args:\n", bold=True),
	pretty_print_list(task["arguments"]["args"], indent=4),
	click.style(" Keyword args:\n", bold=True),
	pretty_print_dict(task["arguments"]["kwargs"], indent=4),
	]

	return "".join(lines)


	@cli.group("task")
	@click.pass_context
	def task(ctx):
	"""Manipulate tasks."""
	pass


	@task.command("schedule")
	@click.option(
	"--columns",
	"-c",
	multiple=True,
	default=["type", "args", "kwargs", "next_run"],
	type=click.Choice(["type", "args", "kwargs", "policy", "next_run"]),
	help="columns present in the CSV file",
	)
	@click.option("--delimiter", "-d", default=",")
	@click.argument("file", type=click.File(encoding="utf-8"))
	@click.pass_context
	def schedule_tasks(ctx, columns, delimiter, file):
	"""Schedule tasks from a CSV input file.

	The following columns are expected, and can be set through the -c option:

	- type: the type of the task to be scheduled (mandatory)

	- args: the arguments passed to the task (JSON list, defaults to an empty
	list)

	- kwargs: the keyword arguments passed to the task (JSON object, defaults
	to an empty dict)

	- next_run: the date at which the task should run (datetime, defaults to
	now)

	The CSV can be read either from a named file, or from stdin (use - as
	filename).

	Use sample:

	cat scheduling-task.txt \| \
	python3 -m swh.scheduler.cli \
	--database 'service=swh-scheduler-dev' \
	task schedule \
	--columns type --columns kwargs --columns policy \
	--delimiter ';' -

	"""
	+ import csv
	+ import json
	+ import arrow
	+
	tasks = []
	now = arrow.utcnow()
	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")

	reader = csv.reader(file, delimiter=delimiter)
	for line in reader:
	task = dict(zip(columns, line))
	args = json.loads(task.pop("args", "[]"))
	kwargs = json.loads(task.pop("kwargs", "{}"))
	task["arguments"] = {
	"args": args,
	"kwargs": kwargs,
	}
	task["next_run"] = DATETIME.convert(task.get("next_run", now), None, None)
	tasks.append(task)

	created = scheduler.create_tasks(tasks)

	output = [
	"Created %d tasks\n" % len(created),
	]
	for task in created:
	output.append(pretty_print_task(task))

	click.echo_via_pager("\n".join(output))


	@task.command("add")
	@click.argument("type", nargs=1, required=True)
	@click.argument("options", nargs=-1)
	@click.option(
	"--policy", "-p", default="recurring", type=click.Choice(["recurring", "oneshot"])
	)
	@click.option(
	"--priority", "-P", default=None, type=click.Choice(["low", "normal", "high"])
	)
	@click.option("--next-run", "-n", default=None)
	@click.pass_context
	def schedule_task(ctx, type, options, policy, priority, next_run):
	"""Schedule one task from arguments.

	The first argument is the name of the task type, further ones are
	positional and keyword argument(s) of the task, in YAML format.
	Keyword args are of the form key=value.

	Usage sample:

	swh-scheduler --database 'service=swh-scheduler' \
	task add list-pypi

	swh-scheduler --database 'service=swh-scheduler' \
	task add list-debian-distribution --policy=oneshot distribution=stretch

	Note: if the priority is not given, the task won't have the priority set,
	which is considered as the lowest priority level.
	"""
	+ import arrow
	+
	from .utils import parse_options

	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")

	now = arrow.utcnow()

	(args, kw) = parse_options(options)
	task = {
	"type": type,
	"policy": policy,
	"priority": priority,
	"arguments": {"args": args, "kwargs": kw,},
	"next_run": DATETIME.convert(next_run or now, None, None),
	}
	created = scheduler.create_tasks([task])

	output = [
	"Created %d tasks\n" % len(created),
	]
	for task in created:
	output.append(pretty_print_task(task))

	click.echo("\n".join(output))


	-def iter_origins(
	- storage: StorageInterface, page_token: Optional[str] = None
	-) -> Iterator[Origin]:
	+def iter_origins( # use string annotations to prevent some pkg loading
	+ storage: "StorageInterface", page_token: "Optional[str]" = None,
	+) -> "Iterator[Origin]":
	"""Iterate over origins in the storage. Optionally starting from page_token.

	This logs regularly an info message during pagination with the page_token. This, in
	order to feed it back to the cli if the process interrupted.

	Yields
	origin model objects from the storage

	"""
	while True:
	page_result = storage.origin_list(page_token=page_token)
	page_token = page_result.next_page_token
	yield from page_result.results
	if not page_token:
	break
	click.echo(f"page_token: {page_token}\n")


	@task.command("schedule_origins")
	@click.argument("type", nargs=1, required=True)
	@click.argument("options", nargs=-1)
	@click.option(
	"--batch-size",
	"-b",
	"origin_batch_size",
	default=10,
	show_default=True,
	type=int,
	help="Number of origins per task",
	)
	@click.option(
	"--page-token",
	default=0,
	show_default=True,
	type=str,
	help="Only schedule tasks for origins whose ID is greater",
	)
	@click.option(
	"--limit",
	default=None,
	type=int,
	help="Limit the tasks scheduling up to this number of tasks",
	)
	@click.option("--storage-url", "-g", help="URL of the (graph) storage API")
	@click.option(
	"--dry-run/--no-dry-run",
	is_flag=True,
	default=False,
	help="List only what would be scheduled.",
	)
	@click.pass_context
	def schedule_origin_metadata_index(
	ctx, type, options, storage_url, origin_batch_size, page_token, limit, dry_run
	):
	"""Schedules tasks for origins that are already known.

	The first argument is the name of the task type, further ones are
	keyword argument(s) of the task in the form key=value, where value is
	in YAML format.

	Usage sample:

	swh-scheduler --database 'service=swh-scheduler' \
	task schedule_origins index-origin-metadata
	"""
	+ from itertools import islice
	from swh.storage import get_storage
	from .utils import parse_options, schedule_origin_batches

	scheduler = ctx.obj["scheduler"]
	storage = get_storage("remote", url=storage_url)
	if dry_run:
	scheduler = None

	(args, kw) = parse_options(options)
	if args:
	raise click.ClickException("Only keywords arguments are allowed.")

	origins = iter_origins(storage, page_token=page_token)
	if limit:
	origins = islice(origins, limit)

	origin_urls = (origin.url for origin in origins)
	schedule_origin_batches(scheduler, type, origin_urls, origin_batch_size, kw)


	@task.command("list-pending")
	@click.argument("task-types", required=True, nargs=-1)
	@click.option(
	"--limit",
	"-l",
	required=False,
	type=click.INT,
	help="The maximum number of tasks to fetch",
	)
	@click.option(
	"--before",
	"-b",
	required=False,
	type=DATETIME,
	help="List all jobs supposed to run before the given date",
	)
	@click.pass_context
	def list_pending_tasks(ctx, task_types, limit, before):
	"""List the tasks that are going to be run.

	You can override the number of tasks to fetch

	"""
	from swh.scheduler import compute_nb_tasks_from

	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")
	num_tasks, num_tasks_priority = compute_nb_tasks_from(limit)

	output = []
	for task_type in task_types:
	pending = scheduler.peek_ready_tasks(
	task_type,
	timestamp=before,
	num_tasks=num_tasks,
	num_tasks_priority=num_tasks_priority,
	)
	output.append("Found %d %s tasks\n" % (len(pending), task_type))

	for task in pending:
	output.append(pretty_print_task(task))

	click.echo("\n".join(output))


	@task.command("list")
	@click.option(
	"--task-id",
	"-i",
	default=None,
	multiple=True,
	metavar="ID",
	help="List only tasks whose id is ID.",
	)
	@click.option(
	"--task-type",
	"-t",
	default=None,
	multiple=True,
	metavar="TYPE",
	help="List only tasks of type TYPE",
	)
	@click.option(
	"--limit",
	"-l",
	required=False,
	type=click.INT,
	help="The maximum number of tasks to fetch.",
	)
	@click.option(
	"--status",
	"-s",
	multiple=True,
	metavar="STATUS",
	type=click.Choice(
	("next_run_not_scheduled", "next_run_scheduled", "completed", "disabled")
	),
	default=None,
	help="List tasks whose status is STATUS.",
	)
	@click.option(
	"--policy",
	"-p",
	default=None,
	type=click.Choice(["recurring", "oneshot"]),
	help="List tasks whose policy is POLICY.",
	)
	@click.option(
	"--priority",
	"-P",
	default=None,
	multiple=True,
	type=click.Choice(["all", "low", "normal", "high"]),
	help="List tasks whose priority is PRIORITY.",
	)
	@click.option(
	"--before",
	"-b",
	required=False,
	type=DATETIME,
	metavar="DATETIME",
	help="Limit to tasks supposed to run before the given date.",
	)
	@click.option(
	"--after",
	"-a",
	required=False,
	type=DATETIME,
	metavar="DATETIME",
	help="Limit to tasks supposed to run after the given date.",
	)
	@click.option(
	"--list-runs",
	"-r",
	is_flag=True,
	default=False,
	help="Also list past executions of each task.",
	)
	@click.pass_context
	def list_tasks(
	ctx, task_id, task_type, limit, status, policy, priority, before, after, list_runs
	):
	"""List tasks.
	"""
	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")

	if not task_type:
	task_type = [x["type"] for x in scheduler.get_task_types()]

	# if task_id is not given, default value for status is
	# 'next_run_not_scheduled'
	# if task_id is given, default status is 'all'
	if task_id is None and status is None:
	status = ["next_run_not_scheduled"]
	if status and "all" in status:
	status = None

	if priority and "all" in priority:
	priority = None

	output = []
	tasks = scheduler.search_tasks(
	task_id=task_id,
	task_type=task_type,
	status=status,
	priority=priority,
	policy=policy,
	before=before,
	after=after,
	limit=limit,
	)
	if list_runs:
	runs = {t["id"]: [] for t in tasks}
	for r in scheduler.get_task_runs([task["id"] for task in tasks]):
	runs[r["task"]].append(r)
	else:
	runs = {}

	output.append("Found %d tasks\n" % (len(tasks)))
	for task in tasks:
	output.append(pretty_print_task(task, full=True))
	if runs.get(task["id"]):
	output.append(click.style(" Executions:", bold=True))
	for run in runs[task["id"]]:
	output.append(pretty_print_run(run, indent=4))

	click.echo("\n".join(output))


	@task.command("respawn")
	@click.argument("task-ids", required=True, nargs=-1)
	@click.option(
	"--next-run",
	"-n",
	required=False,
	type=DATETIME,
	metavar="DATETIME",
	default=None,
	help="Re spawn the selected tasks at this date",
	)
	@click.pass_context
	def respawn_tasks(ctx, task_ids, next_run):
	"""Respawn tasks.

	Respawn tasks given by their ids (see the 'task list' command to
	find task ids) at the given date (immediately by default).

	Eg.

	swh-scheduler task respawn 1 3 12
	"""
	+ import arrow
	+
	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")
	if next_run is None:
	next_run = arrow.utcnow()
	output = []

	scheduler.set_status_tasks(
	task_ids, status="next_run_not_scheduled", next_run=next_run
	)
	output.append("Respawn tasks %s\n" % (task_ids,))

	click.echo("\n".join(output))


	@task.command("archive")
	@click.option(
	"--before",
	"-b",
	default=None,
	help="""Task whose ended date is anterior will be archived.
	Default to current month's first day.""",
	)
	@click.option(
	"--after",
	"-a",
	default=None,
	help="""Task whose ended date is after the specified date will
	be archived. Default to prior month's first day.""",
	)
	@click.option(
	"--batch-index",
	default=1000,
	type=click.INT,
	help="Batch size of tasks to read from db to archive",
	)
	@click.option(
	"--bulk-index",
	default=200,
	type=click.INT,
	help="Batch size of tasks to bulk index",
	)
	@click.option(
	"--batch-clean",
	default=1000,
	type=click.INT,
	help="Batch size of task to clean after archival",
	)
	@click.option(
	"--dry-run/--no-dry-run",
	is_flag=True,
	default=False,
	help="Default to list only what would be archived.",
	)
	@click.option("--verbose", is_flag=True, default=False, help="Verbose mode")
	@click.option(
	"--cleanup/--no-cleanup",
	is_flag=True,
	default=True,
	help="Clean up archived tasks (default)",
	)
	@click.option(
	"--start-from",
	type=click.STRING,
	default=None,
	help="(Optional) default page to start from.",
	)
	@click.pass_context
	def archive_tasks(
	ctx,
	before,
	after,
	batch_index,
	bulk_index,
	batch_clean,
	dry_run,
	verbose,
	cleanup,
	start_from,
	):
	"""Archive task/task_run whose (task_type is 'oneshot' and task_status
	is 'completed') or (task_type is 'recurring' and task_status is
	'disabled').

	With --dry-run flag set (default), only list those.

	"""
	+ import arrow
	+ from itertools import groupby
	from swh.core.utils import grouper
	from swh.scheduler.backend_es import ElasticSearchBackend

	config = ctx.obj["config"]
	scheduler = ctx.obj["scheduler"]

	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")

	logging.basicConfig(level=logging.DEBUG if verbose else logging.INFO)
	logger = logging.getLogger(__name__)
	logging.getLogger("urllib3").setLevel(logging.WARN)
	logging.getLogger("elasticsearch").setLevel(logging.ERROR)
	if dry_run:
	logger.info("DRY-RUN (only reading db)")
	if not cleanup:
	logger.info("NO CLEANUP")

	es_storage = ElasticSearchBackend(**config)
	now = arrow.utcnow()

	# Default to archive tasks from a rolling month starting the week
	# prior to the current one
	if not before:
	before = now.shift(weeks=-1).format("YYYY-MM-DD")

	if not after:
	after = now.shift(weeks=-1).shift(months=-1).format("YYYY-MM-DD")

	logger.debug(
	"index: %s; cleanup: %s; period: [%s ; %s]"
	% (not dry_run, not dry_run and cleanup, after, before)
	)

	def get_index_name(
	data: Dict[str, Any], es_storage: ElasticSearchBackend = es_storage
	) -> str:
	"""Given a data record, determine the index's name through its ending
	date. This varies greatly depending on the task_run's
	status.

	"""
	date = data.get("started")
	if not date:
	date = data["scheduled"]
	return es_storage.compute_index_name(date.year, date.month)

	def index_data(before, page_token, batch_index):
	while True:
	result = scheduler.filter_task_to_archive(
	after, before, page_token=page_token, limit=batch_index
	)
	tasks_sorted = sorted(result["tasks"], key=get_index_name)
	- groups = itertools.groupby(tasks_sorted, key=get_index_name)
	+ groups = groupby(tasks_sorted, key=get_index_name)
	for index_name, tasks_group in groups:
	logger.debug("Index tasks to %s" % index_name)
	if dry_run:
	for task in tasks_group:
	yield task
	continue

	yield from es_storage.streaming_bulk(
	index_name,
	tasks_group,
	source=["task_id", "task_run_id"],
	chunk_size=bulk_index,
	)

	page_token = result.get("next_page_token")
	if page_token is None:
	break

	gen = index_data(before, page_token=start_from, batch_index=batch_index)
	if cleanup:
	for task_ids in grouper(gen, n=batch_clean):
	task_ids = list(task_ids)
	logger.info("Clean up %s tasks: [%s, ...]" % (len(task_ids), task_ids[0]))
	if dry_run: # no clean up
	continue
	ctx.obj["scheduler"].delete_archived_tasks(task_ids)
	else:
	for task_ids in grouper(gen, n=batch_index):
	task_ids = list(task_ids)
	logger.info("Indexed %s tasks: [%s, ...]" % (len(task_ids), task_ids[0]))

	logger.debug("Done!")
	diff --git a/swh/scheduler/cli/task_type.py b/swh/scheduler/cli/task_type.py
	index f4eb546..4320d44 100644
	--- a/swh/scheduler/cli/task_type.py
	+++ b/swh/scheduler/cli/task_type.py
	@@ -1,230 +1,233 @@
	# Copyright (C) 2016-2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-import celery.app.task
	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control
	import click
	import logging

	from importlib import import_module
	from typing import Mapping

	from pkg_resources import iter_entry_points
	from . import cli


	logger = logging.getLogger(__name__)


	DEFAULT_TASK_TYPE = {
	"full": { # for tasks like 'list_xxx_full()'
	"default_interval": "90 days",
	"min_interval": "90 days",
	"max_interval": "90 days",
	"backoff_factor": 1,
	},
	"*": { # value if not suffix matches
	"default_interval": "1 day",
	"min_interval": "1 day",
	"max_interval": "1 day",
	"backoff_factor": 1,
	},
	}


	PLUGIN_WORKER_DESCRIPTIONS = {
	entry_point.name: entry_point for entry_point in iter_entry_points("swh.workers")
	}


	@cli.group("task-type")
	@click.pass_context
	def task_type(ctx):
	"""Manipulate task types."""
	pass


	@task_type.command("list")
	@click.option("--verbose", "-v", is_flag=True, default=False, help="Verbose mode")
	@click.option(
	"--task_type",
	"-t",
	multiple=True,
	default=None,
	help="List task types of given type",
	)
	@click.option(
	"--task_name",
	"-n",
	multiple=True,
	default=None,
	help="List task types of given backend task name",
	)
	@click.pass_context
	def list_task_types(ctx, verbose, task_type, task_name):
	click.echo("Known task types:")
	if verbose:
	tmpl = (
	click.style("{type}: ", bold=True)
	+ """{backend_name}
	{description}
	interval: {default_interval} [{min_interval}, {max_interval}]
	backoff_factor: {backoff_factor}
	max_queue_length: {max_queue_length}
	num_retries: {num_retries}
	retry_delay: {retry_delay}
	"""
	)
	else:
	tmpl = "{type}:\n {description}"
	for tasktype in sorted(
	ctx.obj["scheduler"].get_task_types(), key=lambda x: x["type"]
	):
	if task_type and tasktype["type"] not in task_type:
	continue
	if task_name and tasktype["backend_name"] not in task_name:
	continue
	click.echo(tmpl.format(**tasktype))


	@task_type.command("register")
	@click.option(
	"--plugins",
	"-p",
	"plugins",
	multiple=True,
	default=("all",),
	type=click.Choice(["all"] + list(PLUGIN_WORKER_DESCRIPTIONS)),
	help="Registers task-types for provided plugins. " "Defaults to all",
	)
	@click.pass_context
	def register_task_types(ctx, plugins):
	"""Register missing task-type entries in the scheduler.

	According to declared tasks in each loaded worker (e.g. lister, loader,
	...) plugins.

	"""
	+ import celery.app.task
	+
	scheduler = ctx.obj["scheduler"]

	if plugins == ("all",):
	plugins = list(PLUGIN_WORKER_DESCRIPTIONS)

	for plugin in plugins:
	entrypoint = PLUGIN_WORKER_DESCRIPTIONS[plugin]
	logger.info("Loading entrypoint for plugin %s", plugin)
	registry_entry = entrypoint.load()()

	for task_module in registry_entry["task_modules"]:
	mod = import_module(task_module)
	for task_name in (x for x in dir(mod) if not x.startswith("_")):
	logger.debug("Loading task name %s", task_name)
	taskobj = getattr(mod, task_name)
	if isinstance(taskobj, celery.app.task.Task):
	tt_name = task_name.replace("_", "-")
	task_cfg = registry_entry.get("task_types", {}).get(tt_name, {})
	ensure_task_type(task_module, tt_name, taskobj, task_cfg, scheduler)


	def ensure_task_type(
	task_module: str, task_type: str, swhtask, task_config: Mapping, scheduler
	):
	"""Ensure a given task-type (for the task_module) exists in the scheduler.

	Args:
	task_module: task module we are currently checking for task type
	consistency
	task_type: the type of the task to check/insert (correspond to
	the 'type' field in the db)
	swhtask (SWHTask): the SWHTask instance the task-type correspond to
	task_config: a dict with specific/overloaded values for the
	task-type to be created
	scheduler: the scheduler object used to access the scheduler db

	"""
	for suffix, defaults in DEFAULT_TASK_TYPE.items():
	if task_type.endswith("-" + suffix):
	task_type_dict = defaults.copy()
	break
	else:
	task_type_dict = DEFAULT_TASK_TYPE["*"].copy()

	task_type_dict["type"] = task_type
	task_type_dict["backend_name"] = swhtask.name
	if swhtask.__doc__:
	task_type_dict["description"] = swhtask.__doc__.splitlines()[0]

	task_type_dict.update(task_config)

	current_task_type = scheduler.get_task_type(task_type)
	if current_task_type:
	# Ensure the existing task_type is consistent in the scheduler
	if current_task_type["backend_name"] != task_type_dict["backend_name"]:
	logger.warning(
	"Existing task type %s for module %s has a "
	"different backend name than current "
	"code version provides (%s vs. %s)",
	task_type,
	task_module,
	current_task_type["backend_name"],
	task_type_dict["backend_name"],
	)
	else:
	logger.info("Create task type %s in scheduler", task_type)
	logger.debug(" %s", task_type_dict)
	scheduler.create_task_type(task_type_dict)


	@task_type.command("add")
	@click.argument("type", required=True)
	@click.argument("task-name", required=True)
	@click.argument("description", required=True)
	@click.option(
	"--default-interval",
	"-i",
	default="90 days",
	help='Default interval ("90 days" by default)',
	)
	@click.option(
	"--min-interval",
	default=None,
	help="Minimum interval (default interval if not set)",
	)
	@click.option(
	"--max-interval",
	"-i",
	default=None,
	help="Maximal interval (default interval if not set)",
	)
	@click.option("--backoff-factor", "-f", type=float, default=1, help="Backoff factor")
	@click.pass_context
	def add_task_type(
	ctx,
	type,
	task_name,
	description,
	default_interval,
	min_interval,
	max_interval,
	backoff_factor,
	):
	"""Create a new task type
	"""
	scheduler = ctx.obj["scheduler"]
	if not scheduler:
	raise ValueError("Scheduler class (local/remote) must be instantiated")
	task_type = dict(
	type=type,
	backend_name=task_name,
	description=description,
	default_interval=default_interval,
	min_interval=min_interval,
	max_interval=max_interval,
	backoff_factor=backoff_factor,
	max_queue_length=None,
	num_retries=None,
	retry_delay=None,
	)
	scheduler.create_task_type(task_type)
	click.echo("OK")
	diff --git a/swh/scheduler/cli/utils.py b/swh/scheduler/cli/utils.py
	index 547d70c..2fb02a5 100644
	--- a/swh/scheduler/cli/utils.py
	+++ b/swh/scheduler/cli/utils.py
	@@ -1,86 +1,90 @@
	# Copyright (C) 2019 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information

	-import itertools
	+# WARNING: do not import unnecessary things here to keep cli startup time under
	+# control

	import click
	-import yaml

	-from swh.scheduler.utils import create_task_dict

	TASK_BATCH_SIZE = 1000 # Number of tasks per query to the scheduler


	def schedule_origin_batches(scheduler, task_type, origins, origin_batch_size, kwargs):
	+ from itertools import islice
	+ from swh.scheduler.utils import create_task_dict
	+
	nb_origins = 0
	nb_tasks = 0

	while True:
	task_batch = []
	for _ in range(TASK_BATCH_SIZE):
	# Group origins
	origin_batch = []
	- for origin in itertools.islice(origins, origin_batch_size):
	+ for origin in islice(origins, origin_batch_size):
	origin_batch.append(origin)
	nb_origins += len(origin_batch)
	if not origin_batch:
	break

	# Create a task for these origins
	args = [origin_batch]
	task_dict = create_task_dict(task_type, "oneshot", args, *kwargs)
	task_batch.append(task_dict)

	# Schedule a batch of tasks
	if not task_batch:
	break
	nb_tasks += len(task_batch)
	if scheduler:
	scheduler.create_tasks(task_batch)
	click.echo("Scheduled %d tasks (%d origins)." % (nb_tasks, nb_origins))

	# Print final status.
	if nb_tasks:
	click.echo("Done.")
	else:
	click.echo("Nothing to do (no origin metadata matched the criteria).")


	def parse_argument(option):
	+ import yaml
	+
	try:
	return yaml.safe_load(option)
	except Exception:
	raise click.ClickException("Invalid argument: {}".format(option))


	def parse_options(options):
	"""Parses options from a CLI as YAML and turns it into Python
	args and kwargs.

	>>> parse_options([])
	([], {})
	>>> parse_options(['foo', 'bar'])
	(['foo', 'bar'], {})
	>>> parse_options(['[foo, bar]'])
	([['foo', 'bar']], {})
	>>> parse_options(['"foo"', '"bar"'])
	(['foo', 'bar'], {})
	>>> parse_options(['foo="bar"'])
	([], {'foo': 'bar'})
	>>> parse_options(['"foo"', 'bar="baz"'])
	(['foo'], {'bar': 'baz'})
	>>> parse_options(['42', 'bar=False'])
	([42], {'bar': False})
	>>> parse_options(['42', 'bar=false'])
	([42], {'bar': False})
	>>> parse_options(['42', '"foo'])
	Traceback (most recent call last):
	...
	click.exceptions.ClickException: Invalid argument: "foo
	"""
	kw_pairs = [x.split("=", 1) for x in options if "=" in x]
	args = [parse_argument(x) for x in options if "=" not in x]
	kw = {k: parse_argument(v) for (k, v) in kw_pairs}
	return (args, kw)

File Metadata

Mime Type: text/x-diff
Expires: Fri, Jul 4, 3:43 PM (1 w, 6 d ago)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 3248872

No OneTemporaryActions

View Options

File Metadata

Event Timeline

No OneTemporary
Actions