diff --git a/swh/objstorage/cloud/__init__.py b/swh/objstorage/cloud/__init__.py --- a/swh/objstorage/cloud/__init__.py +++ b/swh/objstorage/cloud/__init__.py @@ -1,3 +1,3 @@ -from objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage +from .objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage __all__ = ['AwsCloudObjStorage', 'OpenStackCloudObjStorage'] diff --git a/swh/objstorage/cloud/objstorage_cloud.py b/swh/objstorage/cloud/objstorage_cloud.py --- a/swh/objstorage/cloud/objstorage_cloud.py +++ b/swh/objstorage/cloud/objstorage_cloud.py @@ -15,14 +15,14 @@ class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta): - """ Abstract ObjStorage that allows connection to a cloud using Libcloud + """Abstract ObjStorage that connect to a cloud using Libcloud - Implementations of this class must redefine the _get_provider method to - make it return a driver provider (i.e. object that supports `get_driver` - method) which return a LibCloud driver - (see https://libcloud.readthedocs.io/en/latest/storage/api.html). - """ + Implementations of this class must redefine the _get_provider + method to make it return a driver provider (i.e. object that + supports `get_driver` method) which return a LibCloud driver (see + https://libcloud.readthedocs.io/en/latest/storage/api.html). + """ def __init__(self, api_key, api_secret_key, container_name): self.driver = self._get_driver(api_key, api_secret_key) self.container_name = container_name @@ -30,7 +30,7 @@ container_name=container_name) def _get_driver(self, api_key, api_secret_key): - """ Initialize a driver to communicate with the cloud + """Initialize a driver to communicate with the cloud Args: api_key: key to connect to the API. @@ -38,6 +38,7 @@ Returns: a Libcloud driver to a cloud storage. + """ # Get the driver class from its description. cls = providers.get_driver(self._get_provider()) @@ -46,12 +47,14 @@ @abc.abstractmethod def _get_provider(self): - """ Get a libcloud driver provider + """Get a libcloud driver provider + + This method must be overriden by subclasses to specify which + of the native libcloud driver the current storage should + connect to. Alternatively, provider for a custom driver may + be returned, in which case the provider will have tu support + `get_driver` method. - This method must be overriden by subclasses to specify which of the - native libcloud driver the current storage should connect to. - Alternatively, provider for a custom driver may be returned, in which - case the provider will have to support `get_driver` method. """ raise NotImplementedError('%s must implement `get_provider` method' % type(self)) @@ -59,7 +62,7 @@ def __contains__(self, obj_id): try: self._get_object(obj_id) - except ObjectDoesNotExistError: + except ObjNotFoundError: return False else: return True @@ -78,13 +81,14 @@ self.driver.iterate_container_objects(self.container)) def __len__(self): - """ Compute the number of objects in the current object storage. + """Compute the number of objects in the current object storage. Warning: this currently uses `__iter__`, its warning about bad performance applies. Returns: number of objects contained in the storage. + """ return sum(1 for i in self) @@ -100,7 +104,7 @@ return obj_id def restore(self, content, obj_id=None): - return self.add(content, obj_id, chech_presence=False) + return self.add(content, obj_id, check_presence=False) def get(self, obj_id): return bytes(self._get_object(obj_id).as_stream()) @@ -115,9 +119,11 @@ raise Error(obj_id) def _get_object(self, obj_id): - """ Get a Libcloud wrapper for an object pointer. + """Get a Libcloud wrapper for an object pointer. + + This wrapper does not retrieve the content of the object + directly. - This wrapper does not retrieve the content of the object directly. """ hex_obj_id = hashutil.hash_to_hex(obj_id) try: @@ -126,10 +132,11 @@ raise ObjNotFoundError(e.object_name) def _put_object(self, content, obj_id): - """ Create an object in the cloud storage. + """Create an object in the cloud storage. + + Created object will contains the content and be referenced by + the given id. - Created object will contain the content and be referenced by the - given id. """ hex_obj_id = hashutil.hash_to_hex(obj_id) self.driver.upload_object_via_stream(iter(content), self.container, @@ -138,6 +145,7 @@ class AwsCloudObjStorage(CloudObjStorage): """ Amazon's S3 Cloud-based object storage + """ def _get_provider(self): return Provider.S3 @@ -145,6 +153,7 @@ class OpenStackCloudObjStorage(CloudObjStorage): """ OpenStack Swift Cloud based object storage + """ def _get_provider(self): return Provider.OPENSTACK_SWIFT diff --git a/swh/objstorage/tests/test_objstorage_cloud.py b/swh/objstorage/tests/test_objstorage_cloud.py new file mode 100644 --- /dev/null +++ b/swh/objstorage/tests/test_objstorage_cloud.py @@ -0,0 +1,87 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import unittest + +from swh.objstorage.cloud.objstorage_cloud import CloudObjStorage +from libcloud.storage.types import (ObjectDoesNotExistError, + ContainerDoesNotExistError) +from libcloud.common.types import InvalidCredsError + +from objstorage_testing import ObjStorageTestFixture + + +API_KEY = 'API_KEY' +API_SECRET_KEY = 'API SECRET KEY' +CONTAINER_NAME = 'test_container' + + +class MockLibcloudObject(): + """ Libcloud object mock that replicates its API """ + def __init__(self, name, content): + self.name = name + self.content = list(content) + + def as_stream(self): + yield from iter(self.content) + + +class MockLibcloudDriver(): + """ Mock driver that replicates the used LibCloud API """ + def __init__(self, api_key, api_secret_key): + self.containers = {CONTAINER_NAME: {}} # Storage is initialized + self.api_key = api_key + self.api_secret_key = api_secret_key + + def _check_credentials(self): + # Private method may be known as another name in Libcloud but is used + # to replicate libcloud behavior (i.e. check credential at each + # request) + if self.api_key != API_KEY or self.api_secret_key != API_SECRET_KEY: + raise InvalidCredsError() + + def get_container(self, container_name): + try: + return self.containers[container_name] + except KeyError: + raise ContainerDoesNotExistError(container_name=container_name, + driver=self, value=None) + + def iterate_container_objects(self, container): + self._check_credentials() + yield from container.values() + + def get_object(self, container_name, obj_id): + self._check_credentials() + try: + container = self.get_container(container_name) + return container[obj_id] + except KeyError: + raise ObjectDoesNotExistError(object_name=obj_id, + driver=self, value=None) + + def upload_object_via_stream(self, content, container, obj_id): + self._check_credentials() + obj = MockLibcloudObject(obj_id, content) + container[obj_id] = obj + + +class MockCloudObjStorage(CloudObjStorage): + """ Cloud object storage that uses a mocked driver """ + def _get_driver(self, api_key, api_secret_key): + return MockLibcloudDriver(api_key, api_secret_key) + + def _get_provider(self): + # Implement this for the abc requirement, but behavior is defined in + # _get_driver. + pass + + +class TestCloudObjStorage(ObjStorageTestFixture, unittest.TestCase): + + def setUp(self): + super().setUp() + self.storage = MockCloudObjStorage(API_KEY, API_SECRET_KEY, + CONTAINER_NAME)