Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/libcloud.py
Show All 9 Lines | |||||
from urllib.parse import urlencode | from urllib.parse import urlencode | ||||
from libcloud.storage import providers | from libcloud.storage import providers | ||||
import libcloud.storage.drivers.s3 | import libcloud.storage.drivers.s3 | ||||
from libcloud.storage.types import ObjectDoesNotExistError, Provider | from libcloud.storage.types import ObjectDoesNotExistError, Provider | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.objstorage.exc import Error, ObjNotFoundError | from swh.objstorage.exc import Error, ObjNotFoundError | ||||
from swh.objstorage.interface import ObjId | |||||
from swh.objstorage.objstorage import ( | from swh.objstorage.objstorage import ( | ||||
ObjStorage, | ObjStorage, | ||||
compressors, | compressors, | ||||
compute_hash, | compute_hash, | ||||
decompressors, | decompressors, | ||||
) | ) | ||||
Show All 30 Lines | Args: | ||||
separated with a slash | separated with a slash | ||||
compression: compression algorithm to use for objects | compression: compression algorithm to use for objects | ||||
kwargs: extra arguments are passed through to the LibCloud driver | kwargs: extra arguments are passed through to the LibCloud driver | ||||
""" | """ | ||||
def __init__( | def __init__( | ||||
self, | self, | ||||
container_name: str, | container_name: str, | ||||
compression: Optional[str] = None, | compression: str = "gzip", | ||||
path_prefix: Optional[str] = None, | path_prefix: Optional[str] = None, | ||||
**kwargs, | **kwargs, | ||||
): | ): | ||||
super().__init__(**kwargs) | super().__init__(**kwargs) | ||||
self.driver = self._get_driver(**kwargs) | self.driver = self._get_driver(**kwargs) | ||||
self.container_name = container_name | self.container_name = container_name | ||||
self.container = self.driver.get_container(container_name=container_name) | self.container = self.driver.get_container(container_name=container_name) | ||||
self.compression = compression | self.compression = compression | ||||
▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | def __len__(self): | ||||
performance applies. | performance applies. | ||||
Returns: | Returns: | ||||
number of objects contained in the storage. | number of objects contained in the storage. | ||||
""" | """ | ||||
return sum(1 for i in self) | return sum(1 for i in self) | ||||
def add(self, content, obj_id, check_presence=True): | def add(self, content: bytes, obj_id: ObjId, check_presence: bool = True) -> ObjId: | ||||
if check_presence and obj_id in self: | if check_presence and obj_id in self: | ||||
return obj_id | return obj_id | ||||
self._put_object(content, obj_id) | self._put_object(content, obj_id) | ||||
return obj_id | return obj_id | ||||
def restore(self, content, obj_id): | def restore(self, content: bytes, obj_id: ObjId): | ||||
return self.add(content, obj_id, check_presence=False) | return self.add(content, obj_id, check_presence=False) | ||||
def get(self, obj_id): | def get(self, obj_id: ObjId) -> bytes: | ||||
obj = b"".join(self._get_object(obj_id).as_stream()) | obj = b"".join(self._get_object(obj_id).as_stream()) | ||||
d = decompressors[self.compression]() | d = decompressors[self.compression]() | ||||
ret = d.decompress(obj) | ret = d.decompress(obj) | ||||
if d.unused_data: | if d.unused_data: | ||||
hex_obj_id = hashutil.hash_to_hex(obj_id) | hex_obj_id = hashutil.hash_to_hex(obj_id) | ||||
raise Error("Corrupt object %s: trailing data found" % hex_obj_id) | raise Error("Corrupt object %s: trailing data found" % hex_obj_id) | ||||
return ret | return ret | ||||
def check(self, obj_id): | def check(self, obj_id: ObjId) -> None: | ||||
# Check that the file exists, as _get_object raises ObjNotFoundError | # Check that the file exists, as _get_object raises ObjNotFoundError | ||||
self._get_object(obj_id) | self._get_object(obj_id) | ||||
# Check the content integrity | # Check the content integrity | ||||
obj_content = self.get(obj_id) | obj_content = self.get(obj_id) | ||||
content_obj_id = compute_hash(obj_content) | content_obj_id = compute_hash(obj_content) | ||||
if content_obj_id != obj_id: | if content_obj_id != obj_id: | ||||
raise Error(obj_id) | raise Error(obj_id) | ||||
def delete(self, obj_id): | def delete(self, obj_id: ObjId): | ||||
super().delete(obj_id) # Check delete permission | super().delete(obj_id) # Check delete permission | ||||
obj = self._get_object(obj_id) | obj = self._get_object(obj_id) | ||||
return self.driver.delete_object(obj) | return self.driver.delete_object(obj) | ||||
def _object_path(self, obj_id): | def _object_path(self, obj_id): | ||||
"""Get the full path to an object""" | """Get the full path to an object""" | ||||
hex_obj_id = hashutil.hash_to_hex(obj_id) | hex_obj_id = hashutil.hash_to_hex(obj_id) | ||||
if self.path_prefix: | if self.path_prefix: | ||||
▲ Show 20 Lines • Show All 56 Lines • Show Last 20 Lines |