Differential D4284 Diff 15182 swh/scheduler/__init__.py

Changeset View

Standalone View

swh/scheduler/init.py

	# Copyright (C) 2018 The Software Heritage developers			# Copyright (C) 2018-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	from typing import Any, Dict			from __future__ import annotations

				from importlib import import_module
				from typing import TYPE_CHECKING, Any, Dict
				import warnings

	# Percentage of tasks with priority to schedule			# Percentage of tasks with priority to schedule
	PRIORITY_SLOT = 0.6			PRIORITY_SLOT = 0.6

	DEFAULT_CONFIG = {			DEFAULT_CONFIG = {
	"scheduler": (			"scheduler": (
	"dict",			"dict",
	{"cls": "local", "args": {"db": "dbname=softwareheritage-scheduler-dev",},},			{"cls": "local", "db": "dbname=softwareheritage-scheduler-dev",},
	)			)
	}			}
	# current configuration. To be set by the config loading mechanism			# current configuration. To be set by the config loading mechanism
	CONFIG = {} # type: Dict[str, Any]			CONFIG = {} # type: Dict[str, Any]


				if TYPE_CHECKING:
				from swh.scheduler.interface import SchedulerInterface


	def compute_nb_tasks_from(num_tasks):			def compute_nb_tasks_from(num_tasks):
	"""Compute and returns the tuple, number of tasks without priority,			"""Compute and returns the tuple, number of tasks without priority,
	number of tasks with priority.			number of tasks with priority.

	Args:			Args:
	num_tasks (int):			num_tasks (int):

	Returns:			Returns:
	tuple number of tasks without priority (int), number of tasks with			tuple number of tasks without priority (int), number of tasks with
	priority (int)			priority (int)

	"""			"""
	if not num_tasks:			if not num_tasks:
	return None, None			return None, None
	return (int((1 - PRIORITY_SLOT) * num_tasks), int(PRIORITY_SLOT * num_tasks))			return (int((1 - PRIORITY_SLOT) * num_tasks), int(PRIORITY_SLOT * num_tasks))


	def get_scheduler(cls, args={}):			BACKEND_TYPES: Dict[str, str] = {
				"local": ".backend.SchedulerBackend",
				"remote": ".api.client.RemoteScheduler",
				}


				def get_scheduler(cls: str, **kwargs) -> SchedulerInterface:
				vlorentzUnsubmitted Not Done Inline Actions they are paths/names, not types. also probably rename "component" to "backend" or "class" vlorentz: they are paths/names, not types. also probably rename "component" to "backend" or "class"
				ardumontAuthorUnsubmitted Done Inline Actions well local and remote are specific backend types for me. It's a map from type to actual fqdn names. What do you think of, BACKEND_TYPES ? (also i'd like to unify that name at some point). ardumont: well local and remote are specific backend types for me. It's a map from type to actual fqdn…
				tenmaUnsubmitted Not Done Inline Actions @vlorentz They are qualified types specified as strings (quoted, a la LISP). Would be equivalent to import beforehand and put the type objects. tenma: @vlorentz They are qualified types specified as strings (quoted, a la LISP). Would be…
	"""			"""
	Get a scheduler object of class `scheduler_class` with arguments			Get a scheduler object of class `cls` with arguments `**kwargs`.
	`scheduler_args`.

	Args:			Args:
	scheduler (dict): dictionary with keys:			cls: scheduler's class, either 'local' or 'remote'
				vlorentzUnsubmitted Not Done Inline Actions s/dictionary with keys, default to empty./arguments to pass to the class' constructor/ vlorentz: s/dictionary with keys, default to empty./arguments to pass to the class' constructor/
				ardumontAuthorUnsubmitted Done Inline Actions thx ardumont: thx
				kwargs: arguments to pass to the class' constructor
	cls (str): scheduler's class, either 'local' or 'remote'
	args (dict): dictionary with keys, default to empty.

	Returns:			Returns:
	an instance of swh.scheduler, either local or remote:			an instance of swh.scheduler, either local or remote:

	local: swh.scheduler.backend.SchedulerBackend			local: swh.scheduler.backend.SchedulerBackend
	remote: swh.scheduler.api.client.RemoteScheduler			remote: swh.scheduler.api.client.RemoteScheduler

	Raises:			Raises:
	ValueError if passed an unknown storage class.			ValueError if passed an unknown storage class.

	"""			"""

	if cls == "remote":			if "args" in kwargs:
	from .api.client import RemoteScheduler as SchedulerBackend			warnings.warn(
	elif cls == "local":			'Explicit "args" key is deprecated, use keys directly instead.',
	from .backend import SchedulerBackend			DeprecationWarning,
	else:			)
	raise ValueError("Unknown swh.scheduler class `%s`" % cls)			kwargs = kwargs["args"]

				class_path = BACKEND_TYPES.get(cls)
				if class_path is None:
				raise ValueError(
				f"Unknown Scheduler class `{cls}`. "
				f"Supported: {', '.join(BACKEND_TYPES)}"
				)

	return SchedulerBackend(**args)			(module_path, class_name) = class_path.rsplit(".", 1)
				module = import_module(module_path, package=__package__)
				BackendClass = getattr(module, class_name)
				return BackendClass(**kwargs)