diff --git a/swh/objstorage/backends/libcloud.py b/swh/objstorage/backends/libcloud.py --- a/swh/objstorage/backends/libcloud.py +++ b/swh/objstorage/backends/libcloud.py @@ -47,12 +47,15 @@ Args: container_name: Name of the base container + path_prefix: prefix to prepend to object paths in the container, + separated with a slash compression: compression algorithm to use for objects kwargs: extra arguments are passed through to the LibCloud driver """ def __init__(self, container_name: str, compression: Optional[str] = None, + path_prefix: Optional[str] = None, **kwargs): super().__init__(**kwargs) self.driver = self._get_driver(**kwargs) @@ -60,6 +63,9 @@ self.container = self.driver.get_container( container_name=container_name) self.compression = compression + self.path_prefix = None + if path_prefix: + self.path_prefix = path_prefix.rstrip('/') + '/' def _get_driver(self, **kwargs): """Initialize a driver to communicate with the cloud @@ -119,8 +125,16 @@ You almost certainly don't want to use this method in production. """ - yield from (hashutil.bytehex_to_hash(obj.name.encode()) for obj in - self.driver.iterate_container_objects(self.container)) + for obj in self.driver.iterate_container_objects(self.container): + name = obj.name + + if self.path_prefix and not name.startswith(self.path_prefix): + continue + + if self.path_prefix: + name = name[len(self.path_prefix):] + + yield hashutil.hash_to_bytes(name) def __len__(self): """Compute the number of objects in the current object storage. @@ -174,7 +188,10 @@ def _object_path(self, obj_id): """Get the full path to an object""" hex_obj_id = hashutil.hash_to_hex(obj_id) - return hex_obj_id + if self.path_prefix: + return self.path_prefix + hex_obj_id + else: + return hex_obj_id def _get_object(self, obj_id): """Get a Libcloud wrapper for an object pointer. diff --git a/swh/objstorage/tests/test_objstorage_cloud.py b/swh/objstorage/tests/test_objstorage_cloud.py --- a/swh/objstorage/tests/test_objstorage_cloud.py +++ b/swh/objstorage/tests/test_objstorage_cloud.py @@ -3,6 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Optional import unittest from libcloud.common.types import InvalidCredsError @@ -93,6 +94,7 @@ class TestCloudObjStorage(ObjStorageTestFixture, unittest.TestCase): compression = 'none' + path_prefix: Optional[str] = None def setUp(self): super().setUp() @@ -100,6 +102,7 @@ CONTAINER_NAME, api_key=API_KEY, api_secret_key=API_SECRET_KEY, compression=self.compression, + path_prefix=self.path_prefix, ) def test_compression(self): @@ -143,3 +146,17 @@ class TestCloudObjStorageZlib(TestCloudObjStorage): compression = 'zlib' + + +class TestCloudObjStoragePrefix(TestCloudObjStorage): + path_prefix = 'contents' + + def test_path_prefix(self): + content, obj_id = self.hash_content(b'test content') + self.storage.add(content, obj_id=obj_id) + + container = self.storage.driver.containers[CONTAINER_NAME] + object_path = self.storage._object_path(obj_id) + + assert object_path.startswith(self.path_prefix + '/') + assert object_path in container