diff --git a/swh/scheduler/cli/task.py b/swh/scheduler/cli/task.py
index 72a1d96..4ee1860 100644
--- a/swh/scheduler/cli/task.py
+++ b/swh/scheduler/cli/task.py
@@ -1,595 +1,599 @@
# Copyright (C) 2016-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from __future__ import annotations
# WARNING: do not import unnecessary things here to keep cli startup time under
# control
import locale
from typing import TYPE_CHECKING, Iterator, List, Optional
import click
from . import cli
if TYPE_CHECKING:
import datetime
# importing swh.storage.interface triggers the load of 300+ modules, so...
import swh.model.model
from swh.storage.interface import StorageInterface
locale.setlocale(locale.LC_ALL, "")
CONTEXT_SETTINGS = dict(help_option_names=["-h", "--help"])
DATETIME = click.DateTime()
def format_dict(d):
"""Recursively format date objects in the dict passed as argument"""
import datetime
ret = {}
for k, v in d.items():
if isinstance(v, (datetime.date, datetime.datetime)):
v = v.isoformat()
elif isinstance(v, dict):
v = format_dict(v)
ret[k] = v
return ret
def pretty_print_list(list, indent=0):
"""Pretty-print a list"""
return "".join("%s%r\n" % (" " * indent, item) for item in list)
def pretty_print_dict(dict, indent=0):
"""Pretty-print a list"""
return "".join(
"%s%s: %r\n" % (" " * indent, click.style(key, bold=True), value)
for key, value in sorted(dict.items())
)
def pretty_print_run(run, indent=4):
fmt = (
"{indent}{backend_id} [{status}]\n"
"{indent} scheduled: {scheduled} [{started}:{ended}]"
)
return fmt.format(indent=" " * indent, **format_dict(run))
def pretty_print_task(task, full=False):
"""Pretty-print a task
If 'full' is True, also print the status and priority fields.
>>> import datetime
>>> task = {
... 'id': 1234,
... 'arguments': {
... 'args': ['foo', 'bar', True],
... 'kwargs': {'key': 'value', 'key2': 42},
... },
... 'current_interval': datetime.timedelta(hours=1),
... 'next_run': datetime.datetime(2019, 2, 21, 13, 52, 35, 407818),
... 'policy': 'oneshot',
... 'priority': None,
... 'status': 'next_run_not_scheduled',
... 'type': 'test_task',
... }
>>> print(click.unstyle(pretty_print_task(task)))
Task 1234
Next run: ... (2019-02-21T13:52:35.407818)
Interval: 1:00:00
Type: test_task
Policy: oneshot
Args:
'foo'
'bar'
True
Keyword args:
key: 'value'
key2: 42
<BLANKLINE>
>>> print(click.unstyle(pretty_print_task(task, full=True)))
Task 1234
Next run: ... (2019-02-21T13:52:35.407818)
Interval: 1:00:00
Type: test_task
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
'foo'
'bar'
True
Keyword args:
key: 'value'
key2: 42
<BLANKLINE>
"""
import humanize
next_run = task["next_run"]
lines = [
"%s %s\n" % (click.style("Task", bold=True), task["id"]),
click.style(" Next run: ", bold=True),
"%s (%s)" % (humanize.naturaldate(next_run), next_run.isoformat()),
"\n",
click.style(" Interval: ", bold=True),
str(task["current_interval"]),
"\n",
click.style(" Type: ", bold=True),
task["type"] or "",
"\n",
click.style(" Policy: ", bold=True),
task["policy"] or "",
"\n",
]
if full:
lines += [
click.style(" Status: ", bold=True),
task["status"] or "",
"\n",
click.style(" Priority: ", bold=True),
task["priority"] or "",
"\n",
]
lines += [
click.style(" Args:\n", bold=True),
pretty_print_list(task["arguments"]["args"], indent=4),
click.style(" Keyword args:\n", bold=True),
pretty_print_dict(task["arguments"]["kwargs"], indent=4),
]
return "".join(lines)
@cli.group("task")
@click.pass_context
def task(ctx):
"""Manipulate tasks."""
pass
@task.command("schedule")
@click.option(
"--columns",
"-c",
multiple=True,
default=["type", "args", "kwargs", "next_run"],
type=click.Choice(["type", "args", "kwargs", "policy", "next_run"]),
help="columns present in the CSV file",
)
@click.option("--delimiter", "-d", default=",")
@click.argument("file", type=click.File(encoding="utf-8"))
@click.pass_context
def schedule_tasks(ctx, columns, delimiter, file):
"""Schedule tasks from a CSV input file.
The following columns are expected, and can be set through the -c option:
- type: the type of the task to be scheduled (mandatory)
- args: the arguments passed to the task (JSON list, defaults to an empty
list)
- kwargs: the keyword arguments passed to the task (JSON object, defaults
to an empty dict)
- next_run: the date at which the task should run (datetime, defaults to
now)
The CSV can be read either from a named file, or from stdin (use - as
filename).
Use sample:
cat scheduling-task.txt | \
python3 -m swh.scheduler.cli \
--database 'service=swh-scheduler-dev' \
task schedule \
--columns type --columns kwargs --columns policy \
--delimiter ';' -
"""
import csv
import json
from swh.scheduler.utils import utcnow
tasks = []
now = utcnow()
scheduler = ctx.obj["scheduler"]
if not scheduler:
raise ValueError("Scheduler class (local/remote) must be instantiated")
reader = csv.reader(file, delimiter=delimiter)
for line in reader:
task = dict(zip(columns, line))
args = json.loads(task.pop("args", "[]"))
kwargs = json.loads(task.pop("kwargs", "{}"))
task["arguments"] = {
"args": args,
"kwargs": kwargs,
}
task["next_run"] = task.get("next_run", now)
tasks.append(task)
created = scheduler.create_tasks(tasks)
output = [
"Created %d tasks\n" % len(created),
]
for task in created:
output.append(pretty_print_task(task))
click.echo_via_pager("\n".join(output))
@task.command("add")
-@click.argument("type", nargs=1, required=True)
+@click.argument("task_type_name", nargs=1, required=True)
@click.argument("options", nargs=-1)
@click.option(
"--policy", "-p", default="recurring", type=click.Choice(["recurring", "oneshot"])
)
@click.option(
"--priority", "-P", default=None, type=click.Choice(["low", "normal", "high"])
)
@click.option("--next-run", "-n", default=None)
@click.pass_context
-def schedule_task(ctx, type, options, policy, priority, next_run):
+def schedule_task(ctx, task_type_name, options, policy, priority, next_run):
"""Schedule one task from arguments.
- The first argument is the name of the task type, further ones are
- positional and keyword argument(s) of the task, in YAML format.
- Keyword args are of the form key=value.
+ The first argument is the name of the task type. Flag options (policy, priority) are
+ task configuration. Further options are positional and keyword argument(s) of the
+ task, in YAML format. Keyword args are of the form key=value.
Usage sample:
swh-scheduler --database 'service=swh-scheduler' \
task add list-pypi
swh-scheduler --database 'service=swh-scheduler' \
task add list-debian-distribution --policy=oneshot distribution=stretch
Note: if the priority is not given, the task won't have the priority set,
which is considered as the lowest priority level.
+
"""
from swh.scheduler.utils import utcnow
from .utils import parse_options
scheduler = ctx.obj["scheduler"]
if not scheduler:
raise ValueError("Scheduler class (local/remote) must be instantiated")
+ if scheduler.get_task_type(task_type_name) is None:
+ raise ValueError(f"Unknown task type {task_type_name}.")
+
now = utcnow()
(args, kw) = parse_options(options)
task = {
- "type": type,
+ "type": task_type_name,
"policy": policy,
"priority": priority,
"arguments": {
"args": args,
"kwargs": kw,
},
"next_run": next_run or now,
}
created = scheduler.create_tasks([task])
output = [
"Created %d tasks\n" % len(created),
]
for task in created:
output.append(pretty_print_task(task))
click.echo("\n".join(output))
def iter_origins( # use string annotations to prevent some pkg loading
storage: StorageInterface,
page_token: Optional[str] = None,
) -> Iterator[swh.model.model.Origin]:
"""Iterate over origins in the storage. Optionally starting from page_token.
This logs regularly an info message during pagination with the page_token. This, in
order to feed it back to the cli if the process interrupted.
Yields
origin model objects from the storage
"""
while True:
page_result = storage.origin_list(page_token=page_token)
page_token = page_result.next_page_token
yield from page_result.results
if not page_token:
break
click.echo(f"page_token: {page_token}\n")
@task.command("schedule_origins")
@click.argument("type", nargs=1, required=True)
@click.argument("options", nargs=-1)
@click.option(
"--batch-size",
"-b",
"origin_batch_size",
default=10,
show_default=True,
type=int,
help="Number of origins per task",
)
@click.option(
"--page-token",
default=0,
show_default=True,
type=str,
help="Only schedule tasks for origins whose ID is greater",
)
@click.option(
"--limit",
default=None,
type=int,
help="Limit the tasks scheduling up to this number of tasks",
)
@click.option("--storage-url", "-g", help="URL of the (graph) storage API")
@click.option(
"--dry-run/--no-dry-run",
is_flag=True,
default=False,
help="List only what would be scheduled.",
)
@click.pass_context
def schedule_origin_metadata_index(
ctx, type, options, storage_url, origin_batch_size, page_token, limit, dry_run
):
"""Schedules tasks for origins that are already known.
The first argument is the name of the task type, further ones are
keyword argument(s) of the task in the form key=value, where value is
in YAML format.
Usage sample:
swh-scheduler --database 'service=swh-scheduler' \
task schedule_origins index-origin-metadata
"""
from itertools import islice
from swh.storage import get_storage
from .utils import parse_options, schedule_origin_batches
scheduler = ctx.obj["scheduler"]
storage = get_storage("remote", url=storage_url)
if dry_run:
scheduler = None
(args, kw) = parse_options(options)
if args:
raise click.ClickException("Only keywords arguments are allowed.")
origins = iter_origins(storage, page_token=page_token)
if limit:
origins = islice(origins, limit)
origin_urls = (origin.url for origin in origins)
schedule_origin_batches(scheduler, type, origin_urls, origin_batch_size, kw)
@task.command("list-pending")
@click.argument("task-types", required=True, nargs=-1)
@click.option(
"--limit",
"-l",
"num_tasks",
required=False,
type=click.INT,
help="The maximum number of tasks to fetch",
)
@click.option(
"--before",
"-b",
required=False,
type=DATETIME,
help="List all jobs supposed to run before the given date",
)
@click.pass_context
def list_pending_tasks(ctx, task_types, num_tasks, before):
"""List tasks with no priority that are going to be run.
You can override the number of tasks to fetch with the --limit flag.
"""
scheduler = ctx.obj["scheduler"]
if not scheduler:
raise ValueError("Scheduler class (local/remote) must be instantiated")
output = []
for task_type in task_types:
pending = scheduler.peek_ready_tasks(
task_type,
timestamp=before,
num_tasks=num_tasks,
)
output.append("Found %d %s tasks\n" % (len(pending), task_type))
for task in pending:
output.append(pretty_print_task(task))
click.echo("\n".join(output))
@task.command("list")
@click.option(
"--task-id",
"-i",
default=None,
multiple=True,
metavar="ID",
help="List only tasks whose id is ID.",
)
@click.option(
"--task-type",
"-t",
default=None,
multiple=True,
metavar="TYPE",
help="List only tasks of type TYPE",
)
@click.option(
"--limit",
"-l",
required=False,
type=click.INT,
help="The maximum number of tasks to fetch.",
)
@click.option(
"--status",
"-s",
multiple=True,
metavar="STATUS",
type=click.Choice(
("next_run_not_scheduled", "next_run_scheduled", "completed", "disabled")
),
default=None,
help="List tasks whose status is STATUS.",
)
@click.option(
"--policy",
"-p",
default=None,
type=click.Choice(["recurring", "oneshot"]),
help="List tasks whose policy is POLICY.",
)
@click.option(
"--priority",
"-P",
default=None,
multiple=True,
type=click.Choice(["all", "low", "normal", "high"]),
help="List tasks whose priority is PRIORITY.",
)
@click.option(
"--before",
"-b",
required=False,
type=DATETIME,
metavar="DATETIME",
help="Limit to tasks supposed to run before the given date.",
)
@click.option(
"--after",
"-a",
required=False,
type=DATETIME,
metavar="DATETIME",
help="Limit to tasks supposed to run after the given date.",
)
@click.option(
"--list-runs",
"-r",
is_flag=True,
default=False,
help="Also list past executions of each task.",
)
@click.pass_context
def list_tasks(
ctx, task_id, task_type, limit, status, policy, priority, before, after, list_runs
):
"""List tasks."""
from operator import itemgetter
scheduler = ctx.obj["scheduler"]
if not scheduler:
raise ValueError("Scheduler class (local/remote) must be instantiated")
if not task_type:
task_type = [x["type"] for x in scheduler.get_task_types()]
# if task_id is not given, default value for status is
# 'next_run_not_scheduled'
# if task_id is given, default status is 'all'
if task_id is None and status is None:
status = ["next_run_not_scheduled"]
if status and "all" in status:
status = None
if priority and "all" in priority:
priority = None
output = []
tasks = scheduler.search_tasks(
task_id=task_id,
task_type=task_type,
status=status,
priority=priority,
policy=policy,
before=before,
after=after,
limit=limit,
)
if list_runs:
runs = {t["id"]: [] for t in tasks}
for r in scheduler.get_task_runs([task["id"] for task in tasks]):
runs[r["task"]].append(r)
else:
runs = {}
output.append("Found %d tasks\n" % (len(tasks)))
for task in sorted(tasks, key=itemgetter("id")):
output.append(pretty_print_task(task, full=True))
if runs.get(task["id"]):
output.append(click.style(" Executions:", bold=True))
for run in sorted(runs[task["id"]], key=itemgetter("id")):
output.append(pretty_print_run(run, indent=4))
click.echo("\n".join(output))
@task.command("respawn")
@click.argument("task-ids", required=True, nargs=-1)
@click.option(
"--next-run",
"-n",
required=False,
type=DATETIME,
metavar="DATETIME",
default=None,
help="Re spawn the selected tasks at this date",
)
@click.pass_context
def respawn_tasks(ctx, task_ids: List[str], next_run: datetime.datetime):
"""Respawn tasks.
Respawn tasks given by their ids (see the 'task list' command to
find task ids) at the given date (immediately by default).
Eg.
swh-scheduler task respawn 1 3 12
"""
from swh.scheduler.utils import utcnow
scheduler = ctx.obj["scheduler"]
if not scheduler:
raise ValueError("Scheduler class (local/remote) must be instantiated")
if next_run is None:
next_run = utcnow()
output = []
task_ids_int = [int(id_) for id_ in task_ids]
scheduler.set_status_tasks(
task_ids_int, status="next_run_not_scheduled", next_run=next_run
)
output.append("Respawn tasks %s\n" % (task_ids_int,))
click.echo("\n".join(output))
diff --git a/swh/scheduler/tests/test_cli.py b/swh/scheduler/tests/test_cli.py
index 6a55071..acf6223 100644
--- a/swh/scheduler/tests/test_cli.py
+++ b/swh/scheduler/tests/test_cli.py
@@ -1,946 +1,960 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import datetime
from itertools import islice
import logging
import random
import re
import tempfile
from unittest.mock import patch
from click.testing import CliRunner
import pytest
from swh.core.api.classes import stream_results
from swh.model.model import Origin
from swh.scheduler.cli import cli
from swh.scheduler.utils import create_task_dict, utcnow
CLI_CONFIG = """
scheduler:
cls: foo
args: {}
"""
def invoke(scheduler, catch_exceptions, args, config=CLI_CONFIG):
runner = CliRunner()
with patch(
"swh.scheduler.get_scheduler"
) as get_scheduler_mock, tempfile.NamedTemporaryFile(
"a", suffix=".yml"
) as config_fd:
config_fd.write(config)
config_fd.seek(0)
get_scheduler_mock.return_value = scheduler
args = [
"-C" + config_fd.name,
] + args
result = runner.invoke(cli, args, obj={"log_level": logging.WARNING})
if not catch_exceptions and result.exception:
print(result.output)
raise result.exception
return result
def test_schedule_tasks(swh_scheduler):
csv_data = (
b'swh-test-ping;[["arg1", "arg2"]];{"key": "value"};'
+ utcnow().isoformat().encode()
+ b"\n"
+ b'swh-test-ping;[["arg3", "arg4"]];{"key": "value"};'
+ utcnow().isoformat().encode()
+ b"\n"
)
with tempfile.NamedTemporaryFile(suffix=".csv") as csv_fd:
csv_fd.write(csv_data)
csv_fd.seek(0)
result = invoke(
swh_scheduler, False, ["task", "schedule", "-d", ";", csv_fd.name]
)
expected = r"""
Created 2 tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: recurring
Args:
\['arg1', 'arg2'\]
Keyword args:
key: 'value'
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: recurring
Args:
\['arg3', 'arg4'\]
Keyword args:
key: 'value'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_schedule_tasks_columns(swh_scheduler):
with tempfile.NamedTemporaryFile(suffix=".csv") as csv_fd:
csv_fd.write(b'swh-test-ping;oneshot;["arg1", "arg2"];{"key": "value"}\n')
csv_fd.seek(0)
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule",
"-c",
"type",
"-c",
"policy",
"-c",
"args",
"-c",
"kwargs",
"-d",
";",
csv_fd.name,
],
)
expected = r"""
Created 1 tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
'arg1'
'arg2'
Keyword args:
key: 'value'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_schedule_task(swh_scheduler):
result = invoke(
swh_scheduler,
False,
[
"task",
"add",
"swh-test-ping",
"arg1",
"arg2",
"key=value",
],
)
expected = r"""
Created 1 tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: recurring
Args:
'arg1'
'arg2'
Keyword args:
key: 'value'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
+def test_schedule_unknown_task_type(swh_scheduler):
+ """When scheduling unknown task type, the cli should raise."""
+ with pytest.raises(ValueError, match="Unknown"):
+ invoke(
+ swh_scheduler,
+ False,
+ [
+ "task",
+ "add",
+ "unknown-task-type-should-raise",
+ ],
+ )
+
+
def test_list_pending_tasks_none(swh_scheduler):
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
],
)
expected = r"""
Found 0 swh-test-ping tasks
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_pending_tasks(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task2["next_run"] += datetime.timedelta(days=1)
swh_scheduler.create_tasks([task1, task2])
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
],
)
expected = r"""
Found 1 swh-test-ping tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value1'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
swh_scheduler.grab_ready_tasks("swh-test-ping")
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
],
)
expected = r"""
Found 0 swh-test-ping tasks
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_pending_tasks_filter(swh_scheduler):
task = create_task_dict("swh-test-multiping", "oneshot", key="value")
swh_scheduler.create_tasks([task])
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
],
)
expected = r"""
Found 0 swh-test-ping tasks
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_pending_tasks_filter_2(swh_scheduler):
swh_scheduler.create_tasks(
[
create_task_dict("swh-test-multiping", "oneshot", key="value"),
create_task_dict("swh-test-ping", "oneshot", key="value2"),
]
)
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
],
)
expected = r"""
Found 1 swh-test-ping tasks
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
# Fails because "task list-pending --limit 3" only returns 2 tasks, because
# of how compute_nb_tasks_from works.
@pytest.mark.xfail
def test_list_pending_tasks_limit(swh_scheduler):
swh_scheduler.create_tasks(
[
create_task_dict("swh-test-ping", "oneshot", key="value%d" % i)
for i in range(10)
]
)
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
"--limit",
"3",
],
)
expected = r"""
Found 2 swh-test-ping tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value0'
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value1'
Task 3
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_pending_tasks_before(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task1["next_run"] += datetime.timedelta(days=3)
task2["next_run"] += datetime.timedelta(days=1)
swh_scheduler.create_tasks([task1, task2])
result = invoke(
swh_scheduler,
False,
[
"task",
"list-pending",
"swh-test-ping",
"--before",
(datetime.date.today() + datetime.timedelta(days=2)).isoformat(),
],
)
expected = r"""
Found 1 swh-test-ping tasks
Task 2
Next run: tomorrow \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task1["next_run"] += datetime.timedelta(days=3, hours=2)
swh_scheduler.create_tasks([task1, task2])
swh_scheduler.grab_ready_tasks("swh-test-ping")
result = invoke(
swh_scheduler,
False,
[
"task",
"list",
],
)
expected = r"""
Found 2 tasks
Task 1
Next run: .+ \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value1'
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_id(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task3 = create_task_dict("swh-test-ping", "oneshot", key="value3")
swh_scheduler.create_tasks([task1, task2, task3])
result = invoke(
swh_scheduler,
False,
[
"task",
"list",
"--task-id",
"2",
],
)
expected = r"""
Found 1 tasks
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_id_2(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task3 = create_task_dict("swh-test-ping", "oneshot", key="value3")
swh_scheduler.create_tasks([task1, task2, task3])
result = invoke(
swh_scheduler, False, ["task", "list", "--task-id", "2", "--task-id", "3"]
)
expected = r"""
Found 2 tasks
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value2'
Task 3
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value3'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_type(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-multiping", "oneshot", key="value2")
task3 = create_task_dict("swh-test-ping", "oneshot", key="value3")
swh_scheduler.create_tasks([task1, task2, task3])
result = invoke(
swh_scheduler, False, ["task", "list", "--task-type", "swh-test-ping"]
)
expected = r"""
Found 2 tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value1'
Task 3
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value3'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_limit(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task3 = create_task_dict("swh-test-ping", "oneshot", key="value3")
swh_scheduler.create_tasks([task1, task2, task3])
result = invoke(
swh_scheduler,
False,
[
"task",
"list",
"--limit",
"2",
],
)
expected = r"""
Found 2 tasks
Task 1
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value1'
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_before(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task1["next_run"] += datetime.timedelta(days=3, hours=2)
swh_scheduler.create_tasks([task1, task2])
swh_scheduler.grab_ready_tasks("swh-test-ping")
result = invoke(
swh_scheduler,
False,
[
"task",
"list",
"--before",
(datetime.date.today() + datetime.timedelta(days=2)).isoformat(),
],
)
expected = r"""
Found 1 tasks
Task 2
Next run: today \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value2'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def test_list_tasks_after(swh_scheduler):
task1 = create_task_dict("swh-test-ping", "oneshot", key="value1")
task2 = create_task_dict("swh-test-ping", "oneshot", key="value2")
task1["next_run"] += datetime.timedelta(days=3, hours=2)
swh_scheduler.create_tasks([task1, task2])
swh_scheduler.grab_ready_tasks("swh-test-ping")
result = invoke(
swh_scheduler,
False,
[
"task",
"list",
"--after",
(datetime.date.today() + datetime.timedelta(days=2)).isoformat(),
],
)
expected = r"""
Found 1 tasks
Task 1
Next run: .+ \(.*\)
Interval: 1 day, 0:00:00
Type: swh-test-ping
Policy: oneshot
Status: next_run_not_scheduled
Priority:\x20
Args:
Keyword args:
key: 'value1'
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), result.output
def _fill_storage_with_origins(storage, nb_origins):
origins = [Origin(url=f"http://example.com/{i}") for i in range(nb_origins)]
storage.origin_add(origins)
return origins
@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3)
def test_task_schedule_origins_dry_run(swh_scheduler, storage):
"""Tests the scheduling when origin_batch_size*task_batch_size is a
divisor of nb_origins."""
_fill_storage_with_origins(storage, 90)
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule_origins",
"--dry-run",
"swh-test-ping",
],
)
# Check the output
expected = r"""
Scheduled 3 tasks \(30 origins\).
Scheduled 6 tasks \(60 origins\).
Scheduled 9 tasks \(90 origins\).
Done.
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), repr(result.output)
# Check scheduled tasks
tasks = swh_scheduler.search_tasks()
assert len(tasks) == 0
def _assert_origin_tasks_contraints(tasks, max_tasks, max_task_size, expected_origins):
# check there are not too many tasks
assert len(tasks) <= max_tasks
# check tasks are not too large
assert all(len(task["arguments"]["args"][0]) <= max_task_size for task in tasks)
# check the tasks are exhaustive
assert sum([len(task["arguments"]["args"][0]) for task in tasks]) == len(
expected_origins
)
assert set.union(*(set(task["arguments"]["args"][0]) for task in tasks)) == {
origin.url for origin in expected_origins
}
@patch("swh.scheduler.cli.utils.TASK_BATCH_SIZE", 3)
def test_task_schedule_origins(swh_scheduler, storage):
"""Tests the scheduling when neither origin_batch_size or
task_batch_size is a divisor of nb_origins."""
origins = _fill_storage_with_origins(storage, 70)
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule_origins",
"swh-test-ping",
"--batch-size",
"20",
],
)
# Check the output
expected = r"""
Scheduled 3 tasks \(60 origins\).
Scheduled 4 tasks \(70 origins\).
Done.
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), repr(result.output)
# Check tasks
tasks = swh_scheduler.search_tasks()
_assert_origin_tasks_contraints(tasks, 4, 20, origins)
assert all(task["arguments"]["kwargs"] == {} for task in tasks)
def test_task_schedule_origins_kwargs(swh_scheduler, storage):
"""Tests support of extra keyword-arguments."""
origins = _fill_storage_with_origins(storage, 30)
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule_origins",
"swh-test-ping",
"--batch-size",
"20",
'key1="value1"',
'key2="value2"',
],
)
# Check the output
expected = r"""
Scheduled 2 tasks \(30 origins\).
Done.
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), repr(result.output)
# Check tasks
tasks = swh_scheduler.search_tasks()
_assert_origin_tasks_contraints(tasks, 2, 20, origins)
assert all(
task["arguments"]["kwargs"] == {"key1": "value1", "key2": "value2"}
for task in tasks
)
def test_task_schedule_origins_with_limit(swh_scheduler, storage):
"""Tests support of extra keyword-arguments."""
_fill_storage_with_origins(storage, 50)
limit = 20
expected_origins = list(islice(stream_results(storage.origin_list), limit))
nb_origins = len(expected_origins)
assert nb_origins == limit
max_task_size = 5
nb_tasks, remainder = divmod(nb_origins, max_task_size)
assert remainder == 0 # made the numbers go round
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule_origins",
"swh-test-ping",
"--batch-size",
max_task_size,
"--limit",
limit,
],
)
# Check the output
expected = rf"""
Scheduled {nb_tasks} tasks \({nb_origins} origins\).
Done.
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), repr(result.output)
tasks = swh_scheduler.search_tasks()
_assert_origin_tasks_contraints(tasks, max_task_size, nb_origins, expected_origins)
def test_task_schedule_origins_with_page_token(swh_scheduler, storage):
"""Tests support of extra keyword-arguments."""
nb_total_origins = 50
origins = _fill_storage_with_origins(storage, nb_total_origins)
# prepare page_token and origins result expectancy
page_result = storage.origin_list(limit=10)
assert len(page_result.results) == 10
page_token = page_result.next_page_token
assert page_token is not None
# remove the first 10 origins listed as we won't see those in tasks
expected_origins = [o for o in origins if o not in page_result.results]
nb_origins = len(expected_origins)
assert nb_origins == nb_total_origins - len(page_result.results)
max_task_size = 10
nb_tasks, remainder = divmod(nb_origins, max_task_size)
assert remainder == 0
result = invoke(
swh_scheduler,
False,
[
"task",
"schedule_origins",
"swh-test-ping",
"--batch-size",
max_task_size,
"--page-token",
page_token,
],
)
# Check the output
expected = rf"""
Scheduled {nb_tasks} tasks \({nb_origins} origins\).
Done.
""".lstrip()
assert result.exit_code == 0, result.output
assert re.fullmatch(expected, result.output, re.MULTILINE), repr(result.output)
# Check tasks
tasks = swh_scheduler.search_tasks()
_assert_origin_tasks_contraints(tasks, max_task_size, nb_origins, expected_origins)
def test_cli_task_runner_unknown_task_types(swh_scheduler, storage):
"""When passing at least one unknown task type, the runner should fail."""
task_types = swh_scheduler.get_task_types()
task_type_names = [t["type"] for t in task_types]
known_task_type = random.choice(task_type_names)
unknown_task_type = "unknown-task-type"
assert unknown_task_type not in task_type_names
with pytest.raises(ValueError, match="Unknown"):
invoke(
swh_scheduler,
False,
[
"start-runner",
"--task-type",
known_task_type,
"--task-type",
unknown_task_type,
],
)
@pytest.mark.parametrize("flag_priority", ["--with-priority", "--without-priority"])
def test_cli_task_runner_with_known_tasks(
swh_scheduler, storage, caplog, flag_priority
):
"""Trigger runner with known tasks runs smoothly."""
task_types = swh_scheduler.get_task_types()
task_type_names = [t["type"] for t in task_types]
task_type_name = random.choice(task_type_names)
task_type_name2 = random.choice(task_type_names)
# The runner will just iterate over the following known tasks and do noop. We are
# just checking the runner does not explode here.
result = invoke(
swh_scheduler,
False,
[
"start-runner",
flag_priority,
"--task-type",
task_type_name,
"--task-type",
task_type_name2,
],
)
assert result.exit_code == 0, result.output
def test_cli_task_runner_no_task(swh_scheduler, storage):
"""Trigger runner with no parameter should run as before."""
# The runner will just iterate over the existing tasks from the scheduler and do
# noop. We are just checking the runner does not explode here.
result = invoke(
swh_scheduler,
False,
[
"start-runner",
],
)
assert result.exit_code == 0, result.output