Differential D2589 Diff 9289 swh/storage/__init__.py

Changeset View

Standalone View

swh/storage/init.py

	# Copyright (C) 2015-2020 The Software Heritage developers			# Copyright (C) 2015-2020 The Software Heritage developers
	# See the AUTHORS file at the top-level directory of this distribution			# See the AUTHORS file at the top-level directory of this distribution
	# License: GNU General Public License version 3, or any later version			# License: GNU General Public License version 3, or any later version
	# See top-level LICENSE file for more information			# See top-level LICENSE file for more information

	import warnings			import warnings

	from . import storage			from . import storage
				from .interface import StorageInterface

	Storage = storage.Storage			Storage = storage.Storage


	class HashCollision(Exception):			class HashCollision(Exception):
	pass			pass


	STORAGE_IMPLEMENTATION = {			STORAGE_IMPLEMENTATION = {
	'pipeline', 'local', 'remote', 'memory', 'filter', 'buffer', 'retry',			'pipeline', 'local', 'remote', 'memory', 'filter', 'buffer', 'retry',
	}			}


	def get_storage(cls, **kwargs):			def get_storage(cls, **kwargs) -> StorageInterface:
	"""Get a storage object of class `storage_class` with arguments			"""Get a storage object of class `storage_class` with arguments
	`storage_args`.			`storage_args`.

	Args:			Args:
	storage (dict): dictionary with keys:			storage (dict): dictionary with keys:
	- cls (str): storage's class, either local, remote, memory, filter,			- cls (str): storage's class, either local, remote, memory, filter,
	buffer			buffer
	- args (dict): dictionary with keys			- args (dict): dictionary with keys

	Returns:			Returns:
	an instance of swh.storage.Storage or compatible class			an instance of swh.storage.Storage or compatible class

	Raises:			Raises:
	ValueError if passed an unknown storage class.			ValueError if passed an unknown storage class.

	"""			"""
	if cls not in STORAGE_IMPLEMENTATION:
	raise ValueError('Unknown storage class `%s`. Supported: %s' % (
	cls, ', '.join(STORAGE_IMPLEMENTATION)))

	if 'args' in kwargs:			if 'args' in kwargs:
	warnings.warn(			warnings.warn(
	'Explicit "args" key is deprecated, use keys directly instead.',			'Explicit "args" key is deprecated, use keys directly instead.',
	DeprecationWarning)			DeprecationWarning)
	kwargs = kwargs['args']			kwargs = kwargs['args']

	if cls == 'pipeline':			if cls == 'pipeline':
	return get_storage_pipeline(**kwargs)			return get_storage_pipeline(**kwargs)

	if cls == 'remote':			elif cls == 'remote':
	from .api.client import RemoteStorage as Storage			from .api.client import RemoteStorage
				return RemoteStorage(**kwargs) # type: ignore # autogenerated methods
	elif cls == 'local':			elif cls == 'local':
	from .storage import Storage			from .storage import Storage
				return Storage(**kwargs)
	elif cls == 'memory':			elif cls == 'memory':
	from .in_memory import InMemoryStorage as Storage			from .in_memory import InMemoryStorage
				return InMemoryStorage(**kwargs)
	elif cls == 'filter':			elif cls == 'filter':
	from .filter import FilteringProxyStorage as Storage			from .filter import FilteringProxyStorage
				return FilteringProxyStorage(**kwargs)
	elif cls == 'buffer':			elif cls == 'buffer':
	from .buffer import BufferingProxyStorage as Storage			from .buffer import BufferingProxyStorage
				return BufferingProxyStorage(**kwargs)
	elif cls == 'retry':			elif cls == 'retry':
	from .retry import RetryingProxyStorage as Storage			from .retry import RetryingProxyStorage
				return RetryingProxyStorage(**kwargs)
	return Storage(**kwargs)			else:
				raise ValueError('Unknown storage class `%s`. Supported: %s' % (
				ardumontUnsubmitted Not Done Inline Actions mmmph, ugly. prefer the old version. Also can't we raise a more explicit message as i remember we do in other part of the stack. raise ValueError('Unsupported storage. Choices are...') or something. ardumont: mmmph, ugly. prefer the old version. Also can't we raise a more explicit message as i remember…
				vlorentzAuthorUnsubmitted Done Inline Actions prefer the old version. mypy says doesn't like the old version. Also can't we raise a more explicit message as i remember we do in other part of the stack. sure vlorentz: > prefer the old version. mypy says doesn't like the old version. > Also can't we raise a…
				cls, ', '.join(STORAGE_IMPLEMENTATION)))


	def get_storage_pipeline(steps):			def get_storage_pipeline(steps):
	"""Recursively get a storage object that may use other storage objects			"""Recursively get a storage object that may use other storage objects
	as backends.			as backends.

	Args:			Args:
	steps (List[dict]): List of dicts that may be used as kwargs for			steps (List[dict]): List of dicts that may be used as kwargs for
	Show All 24 Lines