Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/__init__.py
# Copyright (C) 2018 The Software Heritage developers | # Copyright (C) 2018-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Any, Dict | from __future__ import annotations | ||||
from importlib import import_module | |||||
from typing import TYPE_CHECKING, Any, Dict | |||||
import warnings | |||||
# Percentage of tasks with priority to schedule | # Percentage of tasks with priority to schedule | ||||
PRIORITY_SLOT = 0.6 | PRIORITY_SLOT = 0.6 | ||||
DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
"scheduler": ( | "scheduler": ( | ||||
"dict", | "dict", | ||||
{"cls": "local", "args": {"db": "dbname=softwareheritage-scheduler-dev",},}, | {"cls": "local", "db": "dbname=softwareheritage-scheduler-dev",}, | ||||
) | ) | ||||
} | } | ||||
# current configuration. To be set by the config loading mechanism | # current configuration. To be set by the config loading mechanism | ||||
CONFIG = {} # type: Dict[str, Any] | CONFIG = {} # type: Dict[str, Any] | ||||
if TYPE_CHECKING: | |||||
from swh.scheduler.interface import SchedulerInterface | |||||
def compute_nb_tasks_from(num_tasks): | def compute_nb_tasks_from(num_tasks): | ||||
"""Compute and returns the tuple, number of tasks without priority, | """Compute and returns the tuple, number of tasks without priority, | ||||
number of tasks with priority. | number of tasks with priority. | ||||
Args: | Args: | ||||
num_tasks (int): | num_tasks (int): | ||||
Returns: | Returns: | ||||
tuple number of tasks without priority (int), number of tasks with | tuple number of tasks without priority (int), number of tasks with | ||||
priority (int) | priority (int) | ||||
""" | """ | ||||
if not num_tasks: | if not num_tasks: | ||||
return None, None | return None, None | ||||
return (int((1 - PRIORITY_SLOT) * num_tasks), int(PRIORITY_SLOT * num_tasks)) | return (int((1 - PRIORITY_SLOT) * num_tasks), int(PRIORITY_SLOT * num_tasks)) | ||||
def get_scheduler(cls, args={}): | BACKEND_TYPES: Dict[str, str] = { | ||||
"local": ".backend.SchedulerBackend", | |||||
"remote": ".api.client.RemoteScheduler", | |||||
} | |||||
def get_scheduler(cls: str, **kwargs) -> SchedulerInterface: | |||||
vlorentz: they are paths/names, not types.
also probably rename "component" to "backend" or "class" | |||||
Done Inline Actionswell local and remote are specific backend types for me. What do you think of, BACKEND_TYPES ? ardumont: well local and remote are specific backend types for me.
It's a map from type to actual fqdn… | |||||
Not Done Inline Actions@vlorentz They are qualified types specified as strings (quoted, a la LISP). Would be equivalent to import beforehand and put the type objects. tenma: @vlorentz They are qualified types specified as strings (quoted, a la LISP). Would be… | |||||
""" | """ | ||||
Get a scheduler object of class `scheduler_class` with arguments | Get a scheduler object of class `cls` with arguments `**kwargs`. | ||||
`scheduler_args`. | |||||
Args: | Args: | ||||
scheduler (dict): dictionary with keys: | cls: scheduler's class, either 'local' or 'remote' | ||||
Not Done Inline Actionss/dictionary with keys, default to empty./arguments to pass to the class' constructor/ vlorentz: s/dictionary with keys, default to empty./arguments to pass to the class' constructor/ | |||||
Done Inline Actionsthx ardumont: thx | |||||
kwargs: arguments to pass to the class' constructor | |||||
cls (str): scheduler's class, either 'local' or 'remote' | |||||
args (dict): dictionary with keys, default to empty. | |||||
Returns: | Returns: | ||||
an instance of swh.scheduler, either local or remote: | an instance of swh.scheduler, either local or remote: | ||||
local: swh.scheduler.backend.SchedulerBackend | local: swh.scheduler.backend.SchedulerBackend | ||||
remote: swh.scheduler.api.client.RemoteScheduler | remote: swh.scheduler.api.client.RemoteScheduler | ||||
Raises: | Raises: | ||||
ValueError if passed an unknown storage class. | ValueError if passed an unknown storage class. | ||||
""" | """ | ||||
if cls == "remote": | if "args" in kwargs: | ||||
from .api.client import RemoteScheduler as SchedulerBackend | warnings.warn( | ||||
elif cls == "local": | 'Explicit "args" key is deprecated, use keys directly instead.', | ||||
from .backend import SchedulerBackend | DeprecationWarning, | ||||
else: | ) | ||||
raise ValueError("Unknown swh.scheduler class `%s`" % cls) | kwargs = kwargs["args"] | ||||
class_path = BACKEND_TYPES.get(cls) | |||||
if class_path is None: | |||||
raise ValueError( | |||||
f"Unknown Scheduler class `{cls}`. " | |||||
f"Supported: {', '.join(BACKEND_TYPES)}" | |||||
) | |||||
return SchedulerBackend(**args) | (module_path, class_name) = class_path.rsplit(".", 1) | ||||
module = import_module(module_path, package=__package__) | |||||
BackendClass = getattr(module, class_name) | |||||
return BackendClass(**kwargs) |
they are paths/names, not types.
also probably rename "component" to "backend" or "class"