diff --git a/PKG-INFO b/PKG-INFO index 73486eb..af00b46 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.8 +Version: 0.0.9 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/bin/swh-objstorage-azure b/bin/swh-objstorage-azure new file mode 100755 index 0000000..bdf4ac7 --- /dev/null +++ b/bin/swh-objstorage-azure @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +# NOT FOR PRODUCTION + +import click + +from swh.objstorage import get_objstorage +from swh.objstorage.cloud.objstorage_azure import AzureCloudObjStorage +from swh.core import config, hashutil + + +class AzureAccess(config.SWHConfig): + """This is an orchestration class to try and check objstorage_azure + implementation.""" + + DEFAULT_CONFIG = { + # Output storage + 'storage_account_name': ('str', 'account-name-as-access-key'), + 'storage_secret_key': ('str', 'associated-secret-key'), + 'container_name': ('str', 'sample-container'), + + # Input storage + 'storage': ('dict', + {'cls': 'pathslicing', + 'args': {'root': '/srv/softwareheritage/objects', + 'slicing': '0:2/2:4/4:6'}}), + } + + CONFIG_BASE_FILENAME = 'objstorage/azure' + + def __init__(self): + super().__init__() + self.config = self.parse_config_file() + + container_name = self.config['container_name'] + + self.azure_cloud_storage = AzureCloudObjStorage( + account_name=self.config['storage_account_name'], + api_secret_key=self.config['storage_secret_key'], + container_name=container_name) + + self.read_objstorage = get_objstorage(**self.config['storage']) + + def _to_id(self, hex_obj_id): + return hashutil.hex_to_hash(hex_obj_id) + + def list_contents(self): + for c in self.azure_cloud_storage: + print(c) + + def send_one_content(self, hex_obj_id): + obj_id = self._to_id(hex_obj_id) + obj_content = self.read_objstorage.get(obj_id) + + self.azure_cloud_storage.add(content=obj_content, + obj_id=obj_id) + + def check_integrity(self, hex_obj_id): + obj_id = self._to_id(hex_obj_id) + self.azure_cloud_storage.check(obj_id) # will raise if problem + + def check_presence(self, hex_obj_id): + obj_id = self._to_id(hex_obj_id) + return obj_id in self.azure_cloud_storage + + def download(self, hex_obj_id): + obj_id = self._to_id(hex_obj_id) + return self.azure_cloud_storage.get(obj_id) + + +@click.command() +def tryout(): + obj_azure = AzureAccess() + + # hex_sample_id = '00000008e22217b439f3e582813bd875e7141a0e' + hex_sample_id = '0001001d2879dd009fc11d0c5f0691940989a76b' + + check_presence = obj_azure.check_presence(hex_sample_id) + print('presence first time should be False:', check_presence) + obj_azure.send_one_content(hex_sample_id) + + check_presence = obj_azure.check_presence(hex_sample_id) + print('presence True:', check_presence) + check_presence = obj_azure.check_presence('dfeffffeffff17b439f3e582813bd875e7141a0e') + print('presence False:', check_presence) + + print() + print('Download a blob') + blob_content = obj_azure.download(hex_sample_id) + print(blob_content) + + print() + try: + obj_azure.download(hex_sample_id.replace('0', 'f')) + except: + print('Expected `blob does not exist`!') + + print() + print('blobs:') + obj_azure.list_contents() + + print() + print('content of %s' % hex_sample_id) + print(obj_azure.download(hex_sample_id)) + + obj_azure.check_integrity(hex_sample_id) + +if __name__ == '__main__': + tryout() diff --git a/swh.objstorage.egg-info/PKG-INFO b/swh.objstorage.egg-info/PKG-INFO index 73486eb..af00b46 100644 --- a/swh.objstorage.egg-info/PKG-INFO +++ b/swh.objstorage.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.8 +Version: 0.0.9 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.objstorage.egg-info/SOURCES.txt b/swh.objstorage.egg-info/SOURCES.txt index 8f38ae2..4651671 100644 --- a/swh.objstorage.egg-info/SOURCES.txt +++ b/swh.objstorage.egg-info/SOURCES.txt @@ -1,47 +1,50 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile requirements.txt setup.py version.txt bin/swh-objstorage-add-dir +bin/swh-objstorage-azure bin/swh-objstorage-fsck debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format swh.objstorage.egg-info/PKG-INFO swh.objstorage.egg-info/SOURCES.txt swh.objstorage.egg-info/dependency_links.txt swh.objstorage.egg-info/requires.txt swh.objstorage.egg-info/top_level.txt swh/objstorage/__init__.py swh/objstorage/checker.py swh/objstorage/exc.py swh/objstorage/objstorage.py swh/objstorage/objstorage_pathslicing.py swh/objstorage/api/__init__.py swh/objstorage/api/client.py swh/objstorage/api/common.py swh/objstorage/api/server.py swh/objstorage/cloud/__init__.py +swh/objstorage/cloud/objstorage_azure.py swh/objstorage/cloud/objstorage_cloud.py swh/objstorage/multiplexer/__init__.py swh/objstorage/multiplexer/multiplexer_objstorage.py swh/objstorage/multiplexer/filter/__init__.py swh/objstorage/multiplexer/filter/filter.py swh/objstorage/multiplexer/filter/id_filter.py swh/objstorage/multiplexer/filter/read_write_filter.py swh/objstorage/tests/objstorage_testing.py swh/objstorage/tests/server_testing.py swh/objstorage/tests/test_checker.py swh/objstorage/tests/test_multiplexer_filter.py swh/objstorage/tests/test_objstorage_api.py +swh/objstorage/tests/test_objstorage_azure.py swh/objstorage/tests/test_objstorage_cloud.py swh/objstorage/tests/test_objstorage_instantiation.py swh/objstorage/tests/test_objstorage_multiplexer.py swh/objstorage/tests/test_objstorage_pathslicing.py \ No newline at end of file diff --git a/swh/objstorage/__init__.py b/swh/objstorage/__init__.py index e25ca8f..c62fcf1 100644 --- a/swh/objstorage/__init__.py +++ b/swh/objstorage/__init__.py @@ -1,49 +1,75 @@ from .objstorage import ObjStorage from .objstorage_pathslicing import PathSlicingObjStorage from .api.client import RemoteObjStorage from .multiplexer import MultiplexerObjStorage from .multiplexer.filter import add_filters -__all__ = ['get_objstorage', 'ObjStorage'] +__all__ = ['get_objstorage', 'ObjStorage', 'register_objstorages'] + _STORAGE_CLASSES = { 'pathslicing': PathSlicingObjStorage, 'remote': RemoteObjStorage, } +def register_objstorages(objstorages_map): + """A function to register new objstorage instances. + + This is expected to be called from the client. + + Use example: + from swh.objstorage import register_objstorage, get_objstorage + from .objstorage_cloud import AwsCloudObjStorage + from .objstorage_cloud import OpenStackCloudObjStorage + from .objstorage_azure import AzureCloudObjStorage + + objstorage.register_objstorage({ + 'aws-storage': AwsCloudObjStorage, + 'openstack-storage': OpenStackCloudObjStorage, + 'azure-storage': AzureCloudObjStorage + }) + + # from now on, one can instanciate a new objstorage + get_objstorage('azure-storage', + {'storage-account-name': 'account-name'}...) + + """ + _STORAGE_CLASSES.update(objstorages_map) + + def get_objstorage(cls, args): """ Create an ObjStorage using the given implementation class. Args: cls (str): objstorage class unique key contained in the _STORAGE_CLASSES dict. args (dict): arguments for the required class of objstorage that must match exactly the one in the `__init__` method of the class. Returns: subclass of ObjStorage that match the given `storage_class` argument. Raises: ValueError: if the given storage class is not a valid objstorage key. """ try: return _STORAGE_CLASSES[cls](**args) except KeyError: raise ValueError('Storage class %s does not exists' % cls) def _construct_filtered_objstorage(storage_conf, filters_conf): return add_filters( get_objstorage(**storage_conf), filters_conf ) _STORAGE_CLASSES['filtered'] = _construct_filtered_objstorage def _construct_multiplexer_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return MultiplexerObjStorage(storages) _STORAGE_CLASSES['multiplexer'] = _construct_multiplexer_objstorage diff --git a/swh/objstorage/cloud/__init__.py b/swh/objstorage/cloud/__init__.py index f9965e4..03ffe16 100644 --- a/swh/objstorage/cloud/__init__.py +++ b/swh/objstorage/cloud/__init__.py @@ -1,3 +1,9 @@ from .objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage +from .objstorage_azure import AzureCloudObjStorage -__all__ = ['AwsCloudObjStorage', 'OpenStackCloudObjStorage'] + +__all__ = [ + 'AwsCloudObjStorage', + 'OpenStackCloudObjStorage', + 'AzureCloudObjStorage', +] diff --git a/swh/objstorage/cloud/objstorage_azure.py b/swh/objstorage/cloud/objstorage_azure.py new file mode 100644 index 0000000..0c62e31 --- /dev/null +++ b/swh/objstorage/cloud/objstorage_azure.py @@ -0,0 +1,85 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import gzip + +from swh.core import hashutil +from swh.objstorage.objstorage import ObjStorage, compute_hash +from swh.objstorage.exc import ObjNotFoundError, Error + +from azure.storage.blob import BlockBlobService + + +class AzureCloudObjStorage(ObjStorage): + """ObjStorage with azure abilities + + """ + def __init__(self, account_name, api_secret_key, container_name): + self.block_blob_service = BlockBlobService( + account_name=account_name, + account_key=api_secret_key) + self.container_name = container_name + + def __contains__(self, obj_id): + hex_obj_id = hashutil.hash_to_hex(obj_id) + return self.block_blob_service.exists( + container_name=self.container_name, + blob_name=hex_obj_id) + + def __iter__(self): + """ Iterate over the objects present in the storage + + """ + for obj in self.block_blob_service.list_blobs(self.container_name): + yield obj.name + + def __len__(self): + """Compute the number of objects in the current object storage. + + Returns: + number of objects contained in the storage. + + """ + return sum(1 for i in self) + + def add(self, content, obj_id=None, check_presence=True): + """Add an obj in storage if it's not there already. + + """ + if obj_id is None: + # Checksum is missing, compute it on the fly. + obj_id = compute_hash(content) + + if check_presence and obj_id in self: + return obj_id + + hex_obj_id = hashutil.hash_to_hex(obj_id) + + # Send the gzipped content + self.block_blob_service.create_blob_from_bytes( + container_name=self.container_name, + blob_name=hex_obj_id, + blob=gzip.compress(content)) + + return obj_id + + def restore(self, content, obj_id=None): + return self.add(content, obj_id, check_presence=False) + + def get(self, obj_id): + hex_obj_id = hashutil.hash_to_hex(obj_id) + blob = self.block_blob_service.get_blob_to_bytes( + container_name=self.container_name, + blob_name=hex_obj_id) + if not blob: + raise ObjNotFoundError('Content %s not found!' % hex_obj_id) + return gzip.decompress(blob.content) + + def check(self, obj_id): + # Check the content integrity + obj_content = self.get(obj_id) + content_obj_id = compute_hash(obj_content) + if content_obj_id != obj_id: + raise Error(obj_id) diff --git a/swh/objstorage/cloud/objstorage_cloud.py b/swh/objstorage/cloud/objstorage_cloud.py index 6d169d6..da7fb1d 100644 --- a/swh/objstorage/cloud/objstorage_cloud.py +++ b/swh/objstorage/cloud/objstorage_cloud.py @@ -1,158 +1,158 @@ # Copyright (C) 2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import abc from swh.core import hashutil from swh.objstorage.objstorage import ObjStorage, compute_hash from swh.objstorage.exc import ObjNotFoundError, Error from libcloud.storage import providers from libcloud.storage.types import Provider, ObjectDoesNotExistError class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta): """Abstract ObjStorage that connect to a cloud using Libcloud Implementations of this class must redefine the _get_provider method to make it return a driver provider (i.e. object that supports `get_driver` method) which return a LibCloud driver (see https://libcloud.readthedocs.io/en/latest/storage/api.html). """ def __init__(self, api_key, api_secret_key, container_name): self.driver = self._get_driver(api_key, api_secret_key) self.container_name = container_name self.container = self.driver.get_container( container_name=container_name) def _get_driver(self, api_key, api_secret_key): """Initialize a driver to communicate with the cloud Args: api_key: key to connect to the API. api_secret_key: secret key for authentification. Returns: a Libcloud driver to a cloud storage. """ # Get the driver class from its description. cls = providers.get_driver(self._get_provider()) # Initialize the driver. return cls(api_key, api_secret_key) @abc.abstractmethod def _get_provider(self): """Get a libcloud driver provider This method must be overriden by subclasses to specify which of the native libcloud driver the current storage should connect to. Alternatively, provider for a custom driver may - be returned, in which case the provider will have tu support + be returned, in which case the provider will have to support `get_driver` method. """ raise NotImplementedError('%s must implement `get_provider` method' % type(self)) def __contains__(self, obj_id): try: self._get_object(obj_id) except ObjNotFoundError: return False else: return True def __iter__(self): """ Iterate over the objects present in the storage Warning: Iteration over the contents of a cloud-based object storage may have bad efficiency: due to the very high amount of objects in it and the fact that it is remote, get all the contents of the current object storage may result in a lot of network requests. You almost certainly don't want to use this method in production. """ yield from map(lambda obj: obj.name, self.driver.iterate_container_objects(self.container)) def __len__(self): """Compute the number of objects in the current object storage. Warning: this currently uses `__iter__`, its warning about bad performance applies. Returns: number of objects contained in the storage. """ return sum(1 for i in self) def add(self, content, obj_id=None, check_presence=True): if obj_id is None: # Checksum is missing, compute it on the fly. obj_id = compute_hash(content) if check_presence and obj_id in self: return obj_id self._put_object(content, obj_id) return obj_id def restore(self, content, obj_id=None): return self.add(content, obj_id, check_presence=False) def get(self, obj_id): return bytes(self._get_object(obj_id).as_stream()) def check(self, obj_id): # Check that the file exists, as _get_object raises ObjNotFoundError self._get_object(obj_id) # Check the content integrity obj_content = self.get(obj_id) content_obj_id = compute_hash(obj_content) if content_obj_id != obj_id: raise Error(obj_id) def _get_object(self, obj_id): """Get a Libcloud wrapper for an object pointer. This wrapper does not retrieve the content of the object directly. """ hex_obj_id = hashutil.hash_to_hex(obj_id) try: return self.driver.get_object(self.container_name, hex_obj_id) except ObjectDoesNotExistError as e: raise ObjNotFoundError(e.object_name) def _put_object(self, content, obj_id): """Create an object in the cloud storage. - Created object will contains the content and be referenced by + Created object will contain the content and be referenced by the given id. """ hex_obj_id = hashutil.hash_to_hex(obj_id) self.driver.upload_object_via_stream(iter(content), self.container, hex_obj_id) class AwsCloudObjStorage(CloudObjStorage): """ Amazon's S3 Cloud-based object storage """ def _get_provider(self): return Provider.S3 class OpenStackCloudObjStorage(CloudObjStorage): """ OpenStack Swift Cloud based object storage """ def _get_provider(self): return Provider.OPENSTACK_SWIFT diff --git a/swh/objstorage/objstorage_pathslicing.py b/swh/objstorage/objstorage_pathslicing.py index bd4721f..ca1ac3b 100644 --- a/swh/objstorage/objstorage_pathslicing.py +++ b/swh/objstorage/objstorage_pathslicing.py @@ -1,276 +1,276 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import gzip import tempfile import random from contextlib import contextmanager from swh.core import hashutil from .objstorage import ObjStorage, compute_hash, ID_HASH_ALGO, ID_HASH_LENGTH from .exc import ObjNotFoundError, Error GZIP_BUFSIZ = 1048576 DIR_MODE = 0o755 FILE_MODE = 0o644 @contextmanager def _write_obj_file(hex_obj_id, objstorage): """ Context manager for writing object files to the object storage. During writing, data are written to a temporary file, which is atomically renamed to the right file name after closing. This context manager also takes care of (gzip) compressing the data on the fly. Usage sample: with _write_obj_file(hex_obj_id, objstorage): f.write(obj_data) Yields: a file-like object open for writing bytes. """ # Get the final paths and create the directory if absent. dir = objstorage._obj_dir(hex_obj_id) if not os.path.isdir(dir): os.makedirs(dir, DIR_MODE, exist_ok=True) path = os.path.join(dir, hex_obj_id) # Create a temporary file. (tmp, tmp_path) = tempfile.mkstemp(suffix='.tmp', prefix='hex_obj_id.', dir=dir) # Open the file and yield it for writing. tmp_f = os.fdopen(tmp, 'wb') with gzip.GzipFile(filename=tmp_path, fileobj=tmp_f) as f: yield f # Then close the temporary file and move it to the right directory. tmp_f.close() os.chmod(tmp_path, FILE_MODE) os.rename(tmp_path, path) @contextmanager def _read_obj_file(hex_obj_id, objstorage): """ Context manager for reading object file in the object storage. Usage sample: with _read_obj_file(hex_obj_id, objstorage) as f: b = f.read() Yields: a file-like object open for reading bytes. """ path = objstorage._obj_path(hex_obj_id) with gzip.GzipFile(path, 'rb') as f: yield f class PathSlicingObjStorage(ObjStorage): """ Implementation of the ObjStorage API based on the hash of the content. On disk, an object storage is a directory tree containing files named after their object IDs. An object ID is a checksum of its content, depending on the value of the ID_HASH_ALGO constant (see hashutil for its meaning). To avoid directories that contain too many files, the object storage has a given slicing. Each slicing correspond to a directory that is named according to the hash of its content. So for instance a file with SHA1 34973274ccef6ab4dfaaf86599792fa9c3fe4689 will be stored in the given object storages : - 0:2/2:4/4:6 : 34/97/32/34973274ccef6ab4dfaaf86599792fa9c3fe4689 - 0:1/0:5/ : 3/34973/34973274ccef6ab4dfaaf86599792fa9c3fe4689 The files in the storage are stored in gzipped compressed format. Attributes: root (string): path to the root directory of the storage on the disk. bounds: list of tuples that indicates the beginning and the end of each subdirectory for a content. """ def __init__(self, root, slicing): """ Create an object to access a hash-slicing based object storage. Args: root (string): path to the root directory of the storage on the disk. slicing (string): string that indicates the slicing to perform on the hash of the content to know the path where it should be stored. """ if not os.path.isdir(root): raise ValueError( 'PathSlicingObjStorage root "%s" is not a directory' % root ) self.root = root # Make a list of tuples where each tuple contains the beginning # and the end of each slicing. self.bounds = [ slice(*map(int, sbounds.split(':'))) for sbounds in slicing.split('/') if sbounds ] max_endchar = max(map(lambda bound: bound.stop, self.bounds)) if ID_HASH_LENGTH < max_endchar: raise ValueError( 'Algorithm %s has too short hash for slicing to char %d' % (ID_HASH_ALGO, max_endchar) ) def __contains__(self, obj_id): hex_obj_id = hashutil.hash_to_hex(obj_id) return os.path.exists(self._obj_path(hex_obj_id)) def __iter__(self): """iterate over the object identifiers currently available in the storage Warning: with the current implementation of the object storage, this method will walk the filesystem to list objects, meaning that listing all objects will be very slow for large storages. You almost certainly don't want to use this method in production. Return: iterator over object IDs """ def obj_iterator(): # XXX hackish: it does not verify that the depth of found files # matches the slicing depth of the storage for root, _dirs, files in os.walk(self.root): for f in files: yield bytes.fromhex(f) return obj_iterator() def __len__(self): """compute the number of objects available in the storage Warning: this currently uses `__iter__`, its warning about bad performances applies Return: number of objects contained in the storage """ return sum(1 for i in self) def _obj_dir(self, hex_obj_id): """ Compute the storage directory of an object. See also: PathSlicingObjStorage::_obj_path Args: hex_obj_id: object id as hexlified string. Returns: Path to the directory that contains the required object. """ slices = [hex_obj_id[bound] for bound in self.bounds] return os.path.join(self.root, *slices) def _obj_path(self, hex_obj_id): """ Compute the full path to an object into the current storage. See also: PathSlicingObjStorage::_obj_dir Args: hex_obj_id: object id as hexlified string. Returns: Path to the actual object corresponding to the given id. """ return os.path.join(self._obj_dir(hex_obj_id), hex_obj_id) def add(self, content, obj_id=None, check_presence=True): if obj_id is None: obj_id = compute_hash(content) if check_presence and obj_id in self: - # If the object is already present, return immediatly. + # If the object is already present, return immediately. return obj_id hex_obj_id = hashutil.hash_to_hex(obj_id) with _write_obj_file(hex_obj_id, self) as f: f.write(content) return obj_id def get(self, obj_id): if obj_id not in self: raise ObjNotFoundError(obj_id) # Open the file and return its content as bytes hex_obj_id = hashutil.hash_to_hex(obj_id) with _read_obj_file(hex_obj_id, self) as f: return f.read() def check(self, obj_id): if obj_id not in self: raise ObjNotFoundError(obj_id) hex_obj_id = hashutil.hash_to_hex(obj_id) try: with gzip.open(self._obj_path(hex_obj_id)) as f: length = None if ID_HASH_ALGO.endswith('_git'): # if the hashing algorithm is git-like, we need to know the # content size to hash on the fly. Do a first pass here to # compute the size length = 0 while True: chunk = f.read(GZIP_BUFSIZ) length += len(chunk) if not chunk: break f.rewind() checksums = hashutil._hash_file_obj(f, length, algorithms=[ID_HASH_ALGO]) actual_obj_id = checksums[ID_HASH_ALGO] if obj_id != actual_obj_id: raise Error( 'Corrupt object %s should have id %s' % (hashutil.hash_to_hex(obj_id), hashutil.hash_to_hex(actual_obj_id)) ) except (OSError, IOError): # IOError is for compatibility with older python versions raise Error('Corrupt object %s is not a gzip file' % obj_id) def get_random(self, batch_size): def get_random_content(self, batch_size): """ Get a batch of content inside a single directory. Returns: a tuple (batch size, batch). """ dirs = [] for level in range(len(self.bounds)): path = os.path.join(self.root, *dirs) dir_list = next(os.walk(path))[1] if 'tmp' in dir_list: dir_list.remove('tmp') dirs.append(random.choice(dir_list)) path = os.path.join(self.root, *dirs) content_list = next(os.walk(path))[2] length = min(batch_size, len(content_list)) return length, map(hashutil.hex_to_hash, random.sample(content_list, length)) while batch_size: length, it = get_random_content(self, batch_size) batch_size = batch_size - length yield from it diff --git a/swh/objstorage/tests/test_objstorage_azure.py b/swh/objstorage/tests/test_objstorage_azure.py new file mode 100644 index 0000000..cfd2b56 --- /dev/null +++ b/swh/objstorage/tests/test_objstorage_azure.py @@ -0,0 +1,60 @@ +# Copyright (C) 2016 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import unittest + +from swh.objstorage.cloud.objstorage_azure import AzureCloudObjStorage + +from objstorage_testing import ObjStorageTestFixture + + +class MockBlob(): + """ Libcloud object mock that replicates its API """ + def __init__(self, name, content): + self.name = name + self.content = content + + +class MockBlockBlobService(): + """Mock internal azure library which AzureCloudObjStorage depends upon. + + """ + data = {} + + def __init__(self, account_name, api_secret_key, container_name): + # do not care for the account_name and the api_secret_key here + self.data[container_name] = {} + + def create_blob_from_bytes(self, container_name, blob_name, blob): + self.data[container_name][blob_name] = blob + + def get_blob_to_bytes(self, container_name, blob_name): + if blob_name not in self.data[container_name]: + return None + return MockBlob(name=blob_name, + content=self.data[container_name][blob_name]) + + def exists(self, container_name, blob_name): + return blob_name in self.data[container_name] + + def list_blobs(self, container_name): + for blob_name, content in self.data[container_name].items(): + yield MockBlob(name=blob_name, content=content) + + +class MockAzureCloudObjStorage(AzureCloudObjStorage): + """ Cloud object storage that uses a mocked driver """ + def __init__(self, api_key, api_secret_key, container_name): + self.container_name = container_name + self.block_blob_service = MockBlockBlobService(api_key, api_secret_key, + container_name) + + +class TestAzureCloudObjStorage(ObjStorageTestFixture, unittest.TestCase): + + def setUp(self): + super().setUp() + self.storage = MockAzureCloudObjStorage( + 'account-name', 'api-secret-key', 'container-name') diff --git a/version.txt b/version.txt index 79d2ebe..21d39d7 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.8-0-gea7ba18 \ No newline at end of file +v0.0.9-0-g87c85be \ No newline at end of file