Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7163807
D100.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
8 KB
Subscribers
None
D100.diff
View Options
diff --git a/swh/objstorage/cloud/__init__.py b/swh/objstorage/cloud/__init__.py
--- a/swh/objstorage/cloud/__init__.py
+++ b/swh/objstorage/cloud/__init__.py
@@ -1,3 +1,3 @@
-from objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage
+from .objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage
__all__ = ['AwsCloudObjStorage', 'OpenStackCloudObjStorage']
diff --git a/swh/objstorage/cloud/objstorage_cloud.py b/swh/objstorage/cloud/objstorage_cloud.py
--- a/swh/objstorage/cloud/objstorage_cloud.py
+++ b/swh/objstorage/cloud/objstorage_cloud.py
@@ -15,14 +15,14 @@
class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta):
- """ Abstract ObjStorage that allows connection to a cloud using Libcloud
+ """Abstract ObjStorage that connect to a cloud using Libcloud
- Implementations of this class must redefine the _get_provider method to
- make it return a driver provider (i.e. object that supports `get_driver`
- method) which return a LibCloud driver
- (see https://libcloud.readthedocs.io/en/latest/storage/api.html).
- """
+ Implementations of this class must redefine the _get_provider
+ method to make it return a driver provider (i.e. object that
+ supports `get_driver` method) which return a LibCloud driver (see
+ https://libcloud.readthedocs.io/en/latest/storage/api.html).
+ """
def __init__(self, api_key, api_secret_key, container_name):
self.driver = self._get_driver(api_key, api_secret_key)
self.container_name = container_name
@@ -30,7 +30,7 @@
container_name=container_name)
def _get_driver(self, api_key, api_secret_key):
- """ Initialize a driver to communicate with the cloud
+ """Initialize a driver to communicate with the cloud
Args:
api_key: key to connect to the API.
@@ -38,6 +38,7 @@
Returns:
a Libcloud driver to a cloud storage.
+
"""
# Get the driver class from its description.
cls = providers.get_driver(self._get_provider())
@@ -46,12 +47,14 @@
@abc.abstractmethod
def _get_provider(self):
- """ Get a libcloud driver provider
+ """Get a libcloud driver provider
+
+ This method must be overriden by subclasses to specify which
+ of the native libcloud driver the current storage should
+ connect to. Alternatively, provider for a custom driver may
+ be returned, in which case the provider will have tu support
+ `get_driver` method.
- This method must be overriden by subclasses to specify which of the
- native libcloud driver the current storage should connect to.
- Alternatively, provider for a custom driver may be returned, in which
- case the provider will have to support `get_driver` method.
"""
raise NotImplementedError('%s must implement `get_provider` method'
% type(self))
@@ -59,7 +62,7 @@
def __contains__(self, obj_id):
try:
self._get_object(obj_id)
- except ObjectDoesNotExistError:
+ except ObjNotFoundError:
return False
else:
return True
@@ -78,13 +81,14 @@
self.driver.iterate_container_objects(self.container))
def __len__(self):
- """ Compute the number of objects in the current object storage.
+ """Compute the number of objects in the current object storage.
Warning: this currently uses `__iter__`, its warning about bad
performance applies.
Returns:
number of objects contained in the storage.
+
"""
return sum(1 for i in self)
@@ -100,7 +104,7 @@
return obj_id
def restore(self, content, obj_id=None):
- return self.add(content, obj_id, chech_presence=False)
+ return self.add(content, obj_id, check_presence=False)
def get(self, obj_id):
return bytes(self._get_object(obj_id).as_stream())
@@ -115,9 +119,11 @@
raise Error(obj_id)
def _get_object(self, obj_id):
- """ Get a Libcloud wrapper for an object pointer.
+ """Get a Libcloud wrapper for an object pointer.
+
+ This wrapper does not retrieve the content of the object
+ directly.
- This wrapper does not retrieve the content of the object directly.
"""
hex_obj_id = hashutil.hash_to_hex(obj_id)
try:
@@ -126,10 +132,11 @@
raise ObjNotFoundError(e.object_name)
def _put_object(self, content, obj_id):
- """ Create an object in the cloud storage.
+ """Create an object in the cloud storage.
+
+ Created object will contains the content and be referenced by
+ the given id.
- Created object will contain the content and be referenced by the
- given id.
"""
hex_obj_id = hashutil.hash_to_hex(obj_id)
self.driver.upload_object_via_stream(iter(content), self.container,
@@ -138,6 +145,7 @@
class AwsCloudObjStorage(CloudObjStorage):
""" Amazon's S3 Cloud-based object storage
+
"""
def _get_provider(self):
return Provider.S3
@@ -145,6 +153,7 @@
class OpenStackCloudObjStorage(CloudObjStorage):
""" OpenStack Swift Cloud based object storage
+
"""
def _get_provider(self):
return Provider.OPENSTACK_SWIFT
diff --git a/swh/objstorage/tests/test_objstorage_cloud.py b/swh/objstorage/tests/test_objstorage_cloud.py
new file mode 100644
--- /dev/null
+++ b/swh/objstorage/tests/test_objstorage_cloud.py
@@ -0,0 +1,87 @@
+# Copyright (C) 2016 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import unittest
+
+from swh.objstorage.cloud.objstorage_cloud import CloudObjStorage
+from libcloud.storage.types import (ObjectDoesNotExistError,
+ ContainerDoesNotExistError)
+from libcloud.common.types import InvalidCredsError
+
+from objstorage_testing import ObjStorageTestFixture
+
+
+API_KEY = 'API_KEY'
+API_SECRET_KEY = 'API SECRET KEY'
+CONTAINER_NAME = 'test_container'
+
+
+class MockLibcloudObject():
+ """ Libcloud object mock that replicates its API """
+ def __init__(self, name, content):
+ self.name = name
+ self.content = list(content)
+
+ def as_stream(self):
+ yield from iter(self.content)
+
+
+class MockLibcloudDriver():
+ """ Mock driver that replicates the used LibCloud API """
+ def __init__(self, api_key, api_secret_key):
+ self.containers = {CONTAINER_NAME: {}} # Storage is initialized
+ self.api_key = api_key
+ self.api_secret_key = api_secret_key
+
+ def _check_credentials(self):
+ # Private method may be known as another name in Libcloud but is used
+ # to replicate libcloud behavior (i.e. check credential at each
+ # request)
+ if self.api_key != API_KEY or self.api_secret_key != API_SECRET_KEY:
+ raise InvalidCredsError()
+
+ def get_container(self, container_name):
+ try:
+ return self.containers[container_name]
+ except KeyError:
+ raise ContainerDoesNotExistError(container_name=container_name,
+ driver=self, value=None)
+
+ def iterate_container_objects(self, container):
+ self._check_credentials()
+ yield from container.values()
+
+ def get_object(self, container_name, obj_id):
+ self._check_credentials()
+ try:
+ container = self.get_container(container_name)
+ return container[obj_id]
+ except KeyError:
+ raise ObjectDoesNotExistError(object_name=obj_id,
+ driver=self, value=None)
+
+ def upload_object_via_stream(self, content, container, obj_id):
+ self._check_credentials()
+ obj = MockLibcloudObject(obj_id, content)
+ container[obj_id] = obj
+
+
+class MockCloudObjStorage(CloudObjStorage):
+ """ Cloud object storage that uses a mocked driver """
+ def _get_driver(self, api_key, api_secret_key):
+ return MockLibcloudDriver(api_key, api_secret_key)
+
+ def _get_provider(self):
+ # Implement this for the abc requirement, but behavior is defined in
+ # _get_driver.
+ pass
+
+
+class TestCloudObjStorage(ObjStorageTestFixture, unittest.TestCase):
+
+ def setUp(self):
+ super().setUp()
+ self.storage = MockCloudObjStorage(API_KEY, API_SECRET_KEY,
+ CONTAINER_NAME)
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jan 30, 3:55 PM (1 h, 55 m)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220584
Attached To
D100: Add tests for the cloud object storage
Event Timeline
Log In to Comment