Differential D2236 Diff 7713 swh/objstorage/backends/libcloud.py

Changeset View

Standalone View

swh/objstorage/backends/libcloud.py

Show All 40 Lines	class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta):
"""Abstract ObjStorage that connect to a cloud using Libcloud		"""Abstract ObjStorage that connect to a cloud using Libcloud

Implementations of this class must redefine the _get_provider		Implementations of this class must redefine the _get_provider
method to make it return a driver provider (i.e. object that		method to make it return a driver provider (i.e. object that
supports `get_driver` method) which return a LibCloud driver (see		supports `get_driver` method) which return a LibCloud driver (see
https://libcloud.readthedocs.io/en/latest/storage/api.html).		https://libcloud.readthedocs.io/en/latest/storage/api.html).

Args:		Args:
container_name: Name of the base container		container_name: Name of the base container
		path_prefix: prefix to prepend to object paths in the container,
		vlorentzUnsubmitted Done Inline Actions Should probably mention that the last slashes will be ignored and one will be added. vlorentz: Should probably mention that the last slashes will be ignored and one will be added.
		olasdAuthorUnsubmitted Done Inline Actions That's what I meant by slash-separated; I'll rephrase. olasd: That's what I meant by slash-separated; I'll rephrase.
		separated with a slash
compression: compression algorithm to use for objects		compression: compression algorithm to use for objects
kwargs: extra arguments are passed through to the LibCloud driver		kwargs: extra arguments are passed through to the LibCloud driver
"""		"""
def __init__(self,		def __init__(self,
container_name: str,		container_name: str,
compression: Optional[str] = None,		compression: Optional[str] = None,
		path_prefix: Optional[str] = None,
**kwargs):		**kwargs):
super().__init__(**kwargs)		super().__init__(**kwargs)
self.driver = self._get_driver(**kwargs)		self.driver = self._get_driver(**kwargs)
self.container_name = container_name		self.container_name = container_name
self.container = self.driver.get_container(		self.container = self.driver.get_container(
container_name=container_name)		container_name=container_name)
self.compression = compression		self.compression = compression
		vlorentzUnsubmitted Done Inline Actions Set it to `''` by default and you can drop most of the conditionals below vlorentz: Set it to `''` by default and you can drop most of the conditionals below
		olasdAuthorUnsubmitted Done Inline Actions I don't know how the library copes with accessing objects with a leading '/' and I don't really want to have to find out ;) olasd: I don't know how the library copes with accessing objects with a leading '/' and I don't really…
		vlorentzUnsubmitted Not Done Inline Actions No, I meant this: self.path_prefix = '' if path_prefix: self.path_prefix = path_prefix.rstrip('/') + '/' vlorentz: No, I meant this: ``` self.path_prefix = '' if path_prefix: self.path_prefix = path_prefix.
		self.path_prefix = None
		if path_prefix:
		self.path_prefix = path_prefix.rstrip('/') + '/'

def _get_driver(self, **kwargs):		def _get_driver(self, **kwargs):
"""Initialize a driver to communicate with the cloud		"""Initialize a driver to communicate with the cloud

Kwargs: arguments passed to the StorageDriver class, typically		Kwargs: arguments passed to the StorageDriver class, typically
key: key to connect to the API.		key: key to connect to the API.
secret: secret key for authentication.		secret: secret key for authentication.
secure: (bool) support HTTPS		secure: (bool) support HTTPS
▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines	def __iter__(self):

Warning: Iteration over the contents of a cloud-based object storage		Warning: Iteration over the contents of a cloud-based object storage
may have bad efficiency: due to the very high amount of objects in it		may have bad efficiency: due to the very high amount of objects in it
and the fact that it is remote, get all the contents of the current		and the fact that it is remote, get all the contents of the current
object storage may result in a lot of network requests.		object storage may result in a lot of network requests.

You almost certainly don't want to use this method in production.		You almost certainly don't want to use this method in production.
"""		"""
yield from (hashutil.bytehex_to_hash(obj.name.encode()) for obj in		for obj in self.driver.iterate_container_objects(self.container):
self.driver.iterate_container_objects(self.container))		name = obj.name

		if self.path_prefix and not name.startswith(self.path_prefix):
		continue

		if self.path_prefix:
		name = name[len(self.path_prefix):]

		yield hashutil.hash_to_bytes(name)

def __len__(self):		def __len__(self):
"""Compute the number of objects in the current object storage.		"""Compute the number of objects in the current object storage.

Warning: this currently uses `__iter__`, its warning about bad		Warning: this currently uses `__iter__`, its warning about bad
performance applies.		performance applies.

Returns:		Returns:
Show All 37 Lines	class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta):
def delete(self, obj_id):		def delete(self, obj_id):
super().delete(obj_id) # Check delete permission		super().delete(obj_id) # Check delete permission
obj = self._get_object(obj_id)		obj = self._get_object(obj_id)
return self.driver.delete_object(obj)		return self.driver.delete_object(obj)

def _object_path(self, obj_id):		def _object_path(self, obj_id):
"""Get the full path to an object"""		"""Get the full path to an object"""
hex_obj_id = hashutil.hash_to_hex(obj_id)		hex_obj_id = hashutil.hash_to_hex(obj_id)
		if self.path_prefix:
		return self.path_prefix + hex_obj_id
		else:
return hex_obj_id		return hex_obj_id

def _get_object(self, obj_id):		def _get_object(self, obj_id):
"""Get a Libcloud wrapper for an object pointer.		"""Get a Libcloud wrapper for an object pointer.

This wrapper does not retrieve the content of the object		This wrapper does not retrieve the content of the object
directly.		directly.

"""		"""
▲ Show 20 Lines • Show All 47 Lines • Show Last 20 Lines