Differential D3640 Diff 12852 swh/storage/__init__.py

Changeset View

Standalone View

swh/storage/init.py

# Copyright (C) 2015-2020 The Software Heritage developers		# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

		import importlib
		from typing import Any, Dict, List
import warnings		import warnings

		from .interface import StorageInterface

STORAGE_IMPLEMENTATION = {
"pipeline",		STORAGE_IMPLEMENTATIONS = {
"local",		"local": ".storage.Storage",
"remote",		"remote": ".api.client.RemoteStorage",
"memory",		"memory": ".in_memory.InMemoryStorage",
"filter",		"filter": ".filter.FilteringProxyStorage",
"buffer",		"buffer": ".buffer.BufferingProxyStorage",
"retry",		"retry": ".retry.RetryingProxyStorage",
		ardumontUnsubmitted Not Done Inline Actions What happened to pipeline? It is used! [1] https://forge.softwareheritage.org/source/puppet-swh-site/browse/production/data/defaults.yaml$1167 ardumont: What happened to pipeline? It is used! [1] https://forge.softwareheritage.org/source/puppet…
		vlorentzAuthorUnsubmitted Done Inline Actions I didn't remove it from `get_storage`. vlorentz: I didn't remove it from `get_storage`.
		ardumontUnsubmitted Not Done Inline Actions ah yeah, thanks ;) ardumont: ah yeah, thanks ;)
"cassandra",		"cassandra": ".cassandra.CassandraStorage",
}		}


def get_storage(cls, **kwargs):		def get_storage(cls: str, **kwargs) -> StorageInterface:
		ardumontUnsubmitted Not Done Inline Actions But is this true though? at least filter/buffer/retry are not per say StorageInterface, aren't they? ardumont: But is this true though? at least filter/buffer/retry are not per say StorageInterface, aren't…
		ardumontUnsubmitted Not Done Inline Actions I'm doing it heh but i keep having that voice raising in my head saying "lies" ;) ardumont: I'm doing it heh but i keep having that voice raising in my head saying "lies" ;)
		vlorentzAuthorUnsubmitted Done Inline Actions They behave differently, but their signatures are the same afaik vlorentz: They behave differently, but their signatures are the same afaik
		ardumontUnsubmitted Not Done Inline Actions yes, indeed. just raising this "concern" to someone else ;) ardumont: yes, indeed. just raising this "concern" to someone else ;)
"""Get a storage object of class `storage_class` with arguments		"""Get a storage object of class `storage_class` with arguments
`storage_args`.		`storage_args`.

Args:		Args:
storage (dict): dictionary with keys:		storage (dict): dictionary with keys:
- cls (str): storage's class, either local, remote, memory, filter,		- cls (str): storage's class, either local, remote, memory, filter,
buffer		buffer
- args (dict): dictionary with keys		- args (dict): dictionary with keys

Returns:		Returns:
an instance of swh.storage.Storage or compatible class		an instance of swh.storage.Storage or compatible class

Raises:		Raises:
ValueError if passed an unknown storage class.		ValueError if passed an unknown storage class.

"""		"""
if cls not in STORAGE_IMPLEMENTATION:
raise ValueError(
"Unknown storage class `%s`. Supported: %s"
% (cls, ", ".join(STORAGE_IMPLEMENTATION))
)

if "args" in kwargs:		if "args" in kwargs:
warnings.warn(		warnings.warn(
'Explicit "args" key is deprecated, use keys directly instead.',		'Explicit "args" key is deprecated, use keys directly instead.',
DeprecationWarning,		DeprecationWarning,
)		)
kwargs = kwargs["args"]		kwargs = kwargs["args"]

if cls == "pipeline":		if cls == "pipeline":
return get_storage_pipeline(**kwargs)		return get_storage_pipeline(**kwargs)

if cls == "remote":		class_path = STORAGE_IMPLEMENTATIONS.get(cls)
from .api.client import RemoteStorage as Storage		if class_path is None:
elif cls == "local":		raise ValueError(
from .storage import Storage		"Unknown storage class `%s`. Supported: %s"
elif cls == "cassandra":		% (cls, ", ".join(STORAGE_IMPLEMENTATIONS))
from .cassandra import CassandraStorage as Storage		)
elif cls == "memory":
from .in_memory import InMemoryStorage as Storage
elif cls == "filter":
from .filter import FilteringProxyStorage as Storage
elif cls == "buffer":
from .buffer import BufferingProxyStorage as Storage
elif cls == "retry":
from .retry import RetryingProxyStorage as Storage

		(module_path, class_name) = class_path.rsplit(".", 1)
		module = importlib.import_module(module_path, package=__package__)
		Storage = getattr(module, class_name)
return Storage(**kwargs)		return Storage(**kwargs)


def get_storage_pipeline(steps):		def get_storage_pipeline(steps: List[Dict[str, Any]]) -> StorageInterface:
		ardumontUnsubmitted Not Done Inline Actions well, might as well type it as well to return a StorageInterface. ardumont: well, might as well type it as well to return a StorageInterface.
"""Recursively get a storage object that may use other storage objects		"""Recursively get a storage object that may use other storage objects
as backends.		as backends.

Args:		Args:
steps (List[dict]): List of dicts that may be used as kwargs for		steps (List[dict]): List of dicts that may be used as kwargs for
`get_storage`.		`get_storage`.

Returns:		Returns:
Show All 12 Lines	for step in reversed(steps):
step = {		step = {
"cls": step["cls"],		"cls": step["cls"],
**step["args"],		**step["args"],
}		}
if storage_config:		if storage_config:
step["storage"] = storage_config		step["storage"] = storage_config
storage_config = step		storage_config = step

		if storage_config is None:
		raise ValueError("'pipeline' has no steps.")

return get_storage(**storage_config)		return get_storage(**storage_config)