Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/backends/libcloud.py
| Show All 9 Lines | |||||
| from urllib.parse import urlencode | from urllib.parse import urlencode | ||||
| from libcloud.storage import providers | from libcloud.storage import providers | ||||
| import libcloud.storage.drivers.s3 | import libcloud.storage.drivers.s3 | ||||
| from libcloud.storage.types import ObjectDoesNotExistError, Provider | from libcloud.storage.types import ObjectDoesNotExistError, Provider | ||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from swh.objstorage.exc import Error, ObjNotFoundError | from swh.objstorage.exc import Error, ObjNotFoundError | ||||
| from swh.objstorage.interface import ObjId | |||||
| from swh.objstorage.objstorage import ( | from swh.objstorage.objstorage import ( | ||||
| ObjStorage, | ObjStorage, | ||||
| compressors, | compressors, | ||||
| compute_hash, | compute_hash, | ||||
| decompressors, | decompressors, | ||||
| ) | ) | ||||
| Show All 30 Lines | Args: | ||||
| separated with a slash | separated with a slash | ||||
| compression: compression algorithm to use for objects | compression: compression algorithm to use for objects | ||||
| kwargs: extra arguments are passed through to the LibCloud driver | kwargs: extra arguments are passed through to the LibCloud driver | ||||
| """ | """ | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| container_name: str, | container_name: str, | ||||
| compression: Optional[str] = None, | compression: str = "gzip", | ||||
| path_prefix: Optional[str] = None, | path_prefix: Optional[str] = None, | ||||
| **kwargs, | **kwargs, | ||||
| ): | ): | ||||
| super().__init__(**kwargs) | super().__init__(**kwargs) | ||||
| self.driver = self._get_driver(**kwargs) | self.driver = self._get_driver(**kwargs) | ||||
| self.container_name = container_name | self.container_name = container_name | ||||
| self.container = self.driver.get_container(container_name=container_name) | self.container = self.driver.get_container(container_name=container_name) | ||||
| self.compression = compression | self.compression = compression | ||||
| ▲ Show 20 Lines • Show All 78 Lines • ▼ Show 20 Lines | def __len__(self): | ||||
| performance applies. | performance applies. | ||||
| Returns: | Returns: | ||||
| number of objects contained in the storage. | number of objects contained in the storage. | ||||
| """ | """ | ||||
| return sum(1 for i in self) | return sum(1 for i in self) | ||||
| def add(self, content, obj_id, check_presence=True): | def add(self, content: bytes, obj_id: ObjId, check_presence: bool = True) -> ObjId: | ||||
| if check_presence and obj_id in self: | if check_presence and obj_id in self: | ||||
| return obj_id | return obj_id | ||||
| self._put_object(content, obj_id) | self._put_object(content, obj_id) | ||||
| return obj_id | return obj_id | ||||
| def restore(self, content, obj_id): | def restore(self, content: bytes, obj_id: ObjId): | ||||
| return self.add(content, obj_id, check_presence=False) | return self.add(content, obj_id, check_presence=False) | ||||
| def get(self, obj_id): | def get(self, obj_id: ObjId) -> bytes: | ||||
| obj = b"".join(self._get_object(obj_id).as_stream()) | obj = b"".join(self._get_object(obj_id).as_stream()) | ||||
| d = decompressors[self.compression]() | d = decompressors[self.compression]() | ||||
| ret = d.decompress(obj) | ret = d.decompress(obj) | ||||
| if d.unused_data: | if d.unused_data: | ||||
| hex_obj_id = hashutil.hash_to_hex(obj_id) | hex_obj_id = hashutil.hash_to_hex(obj_id) | ||||
| raise Error("Corrupt object %s: trailing data found" % hex_obj_id) | raise Error("Corrupt object %s: trailing data found" % hex_obj_id) | ||||
| return ret | return ret | ||||
| def check(self, obj_id): | def check(self, obj_id: ObjId) -> None: | ||||
| # Check that the file exists, as _get_object raises ObjNotFoundError | # Check that the file exists, as _get_object raises ObjNotFoundError | ||||
| self._get_object(obj_id) | self._get_object(obj_id) | ||||
| # Check the content integrity | # Check the content integrity | ||||
| obj_content = self.get(obj_id) | obj_content = self.get(obj_id) | ||||
| content_obj_id = compute_hash(obj_content) | content_obj_id = compute_hash(obj_content) | ||||
| if content_obj_id != obj_id: | if content_obj_id != obj_id: | ||||
| raise Error(obj_id) | raise Error(obj_id) | ||||
| def delete(self, obj_id): | def delete(self, obj_id: ObjId): | ||||
| super().delete(obj_id) # Check delete permission | super().delete(obj_id) # Check delete permission | ||||
| obj = self._get_object(obj_id) | obj = self._get_object(obj_id) | ||||
| return self.driver.delete_object(obj) | return self.driver.delete_object(obj) | ||||
| def _object_path(self, obj_id): | def _object_path(self, obj_id): | ||||
| """Get the full path to an object""" | """Get the full path to an object""" | ||||
| hex_obj_id = hashutil.hash_to_hex(obj_id) | hex_obj_id = hashutil.hash_to_hex(obj_id) | ||||
| if self.path_prefix: | if self.path_prefix: | ||||
| ▲ Show 20 Lines • Show All 56 Lines • Show Last 20 Lines | |||||