Index: swh/objstorage/__init__.py =================================================================== --- swh/objstorage/__init__.py +++ swh/objstorage/__init__.py @@ -3,14 +3,14 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from .objstorage import ObjStorage -from .objstorage_pathslicing import PathSlicingObjStorage -from .objstorage_in_memory import InMemoryObjStorage -from .api.client import RemoteObjStorage -from .multiplexer import MultiplexerObjStorage, StripingObjStorage -from .multiplexer.filter import add_filters - -from swh.objstorage.objstorage_weed import WeedObjStorage +from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa +from swh.objstorage.backends.pathslicing import PathSlicingObjStorage +from swh.objstorage.backends.in_memory import InMemoryObjStorage +from swh.objstorage.api.client import RemoteObjStorage +from swh.objstorage.multiplexer import ( + MultiplexerObjStorage, StripingObjStorage) +from swh.objstorage.multiplexer.filter import add_filters +from swh.objstorage.backends.seaweed import WeedObjStorage __all__ = ['get_objstorage', 'ObjStorage'] @@ -26,7 +26,7 @@ } try: - from swh.objstorage.cloud.objstorage_azure import ( + from swh.objstorage.backends.azure import ( AzureCloudObjStorage, PrefixedAzureCloudObjStorage, ) @@ -37,13 +37,13 @@ _STORAGE_CLASSES_MISSING['azure-prefixed'] = e.args[0] try: - from swh.objstorage.objstorage_rados import RADOSObjStorage + from swh.objstorage.backends.rados import RADOSObjStorage _STORAGE_CLASSES['rados'] = RADOSObjStorage except ImportError as e: _STORAGE_CLASSES_MISSING['rados'] = e.args[0] try: - from swh.objstorage.cloud.objstorage_cloud import ( + from swh.objstorage.backends.libcloud import ( AwsCloudObjStorage, OpenStackCloudObjStorage, ) Index: swh/objstorage/backends/pathslicing.py =================================================================== --- swh/objstorage/backends/pathslicing.py +++ swh/objstorage/backends/pathslicing.py @@ -15,9 +15,10 @@ from swh.model import hashutil -from .objstorage import (ObjStorage, compute_hash, ID_HASH_ALGO, - ID_HASH_LENGTH, DEFAULT_CHUNK_SIZE, DEFAULT_LIMIT) -from .exc import ObjNotFoundError, Error +from swh.objstorage.objstorage import ( + ObjStorage, compute_hash, ID_HASH_ALGO, + ID_HASH_LENGTH, DEFAULT_CHUNK_SIZE, DEFAULT_LIMIT) +from swh.objstorage.exc import ObjNotFoundError, Error GZIP_BUFSIZ = 1048576 Index: swh/objstorage/cloud/__init__.py =================================================================== --- swh/objstorage/cloud/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .objstorage_cloud import AwsCloudObjStorage, OpenStackCloudObjStorage -from .objstorage_azure import AzureCloudObjStorage - - -__all__ = [ - 'AwsCloudObjStorage', - 'OpenStackCloudObjStorage', - 'AzureCloudObjStorage', -] Index: swh/objstorage/cloud/objstorage_azure.py =================================================================== --- /dev/null +++ swh/objstorage/cloud/objstorage_azure.py @@ -1,225 +0,0 @@ -# Copyright (C) 2016-2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import gzip -import string -from itertools import dropwhile, islice, product - -from azure.storage.blob import BlockBlobService -from azure.common import AzureMissingResourceHttpError -import requests - -from swh.objstorage.objstorage import ObjStorage, compute_hash, DEFAULT_LIMIT -from swh.objstorage.exc import ObjNotFoundError, Error -from swh.model import hashutil - - -class AzureCloudObjStorage(ObjStorage): - """ObjStorage with azure abilities. - - """ - def __init__(self, account_name, api_secret_key, container_name, **kwargs): - super().__init__(**kwargs) - self.block_blob_service = BlockBlobService( - account_name=account_name, - account_key=api_secret_key, - request_session=requests.Session(), - ) - self.container_name = container_name - - def get_blob_service(self, hex_obj_id): - """Get the block_blob_service and container that contains the object with - internal id hex_obj_id - """ - return self.block_blob_service, self.container_name - - def get_all_blob_services(self): - """Get all active block_blob_services""" - yield self.block_blob_service, self.container_name - - def _internal_id(self, obj_id): - """Internal id is the hex version in objstorage. - - """ - return hashutil.hash_to_hex(obj_id) - - def check_config(self, *, check_write): - """Check the configuration for this object storage""" - for service, container in self.get_all_blob_services(): - props = service.get_container_properties(container) - - # FIXME: check_write is ignored here - if not props: - return False - - return True - - def __contains__(self, obj_id): - """Does the storage contains the obj_id. - - """ - hex_obj_id = self._internal_id(obj_id) - service, container = self.get_blob_service(hex_obj_id) - return service.exists( - container_name=container, - blob_name=hex_obj_id) - - def __iter__(self): - """Iterate over the objects present in the storage. - - """ - for service, container in self.get_all_blob_services(): - for obj in service.list_blobs(container): - yield hashutil.hash_to_bytes(obj.name) - - def __len__(self): - """Compute the number of objects in the current object storage. - - Returns: - number of objects contained in the storage. - - """ - return sum(1 for i in self) - - def add(self, content, obj_id=None, check_presence=True): - """Add an obj in storage if it's not there already. - - """ - if obj_id is None: - # Checksum is missing, compute it on the fly. - obj_id = compute_hash(content) - - if check_presence and obj_id in self: - return obj_id - - hex_obj_id = self._internal_id(obj_id) - - # Send the gzipped content - service, container = self.get_blob_service(hex_obj_id) - service.create_blob_from_bytes( - container_name=container, - blob_name=hex_obj_id, - blob=gzip.compress(content)) - - return obj_id - - def restore(self, content, obj_id=None): - """Restore a content. - - """ - return self.add(content, obj_id, check_presence=False) - - def get(self, obj_id): - """Retrieve blob's content if found. - - """ - hex_obj_id = self._internal_id(obj_id) - service, container = self.get_blob_service(hex_obj_id) - try: - blob = service.get_blob_to_bytes( - container_name=container, - blob_name=hex_obj_id) - except AzureMissingResourceHttpError: - raise ObjNotFoundError(obj_id) - - return gzip.decompress(blob.content) - - def check(self, obj_id): - """Check the content integrity. - - """ - obj_content = self.get(obj_id) - content_obj_id = compute_hash(obj_content) - if content_obj_id != obj_id: - raise Error(obj_id) - - def delete(self, obj_id): - """Delete an object.""" - super().delete(obj_id) # Check delete permission - hex_obj_id = self._internal_id(obj_id) - service, container = self.get_blob_service(hex_obj_id) - try: - service.delete_blob( - container_name=container, - blob_name=hex_obj_id) - except AzureMissingResourceHttpError: - raise ObjNotFoundError('Content {} not found!'.format(hex_obj_id)) - - return True - - def list_content(self, last_obj_id=None, limit=DEFAULT_LIMIT): - all_blob_services = self.get_all_blob_services() - if last_obj_id: - last_obj_id = self._internal_id(last_obj_id) - last_service, _ = self.get_blob_service(last_obj_id) - all_blob_services = dropwhile( - lambda srv: srv[0] != last_service, all_blob_services) - else: - last_service = None - - def iterate_blobs(): - for service, container in all_blob_services: - marker = last_obj_id if service == last_service else None - for obj in service.list_blobs( - container, marker=marker, maxresults=limit): - yield hashutil.hash_to_bytes(obj.name) - return islice(iterate_blobs(), limit) - - -class PrefixedAzureCloudObjStorage(AzureCloudObjStorage): - """ObjStorage with azure capabilities, striped by prefix. - - accounts is a dict containing entries of the form: - : - account_name: - api_secret_key: - container_name: - """ - def __init__(self, accounts, **kwargs): - # shortcut AzureCloudObjStorage __init__ - ObjStorage.__init__(self, **kwargs) - - # Definition sanity check - prefix_lengths = set(len(prefix) for prefix in accounts) - if not len(prefix_lengths) == 1: - raise ValueError("Inconsistent prefixes, found lengths %s" - % ', '.join( - str(l) for l in sorted(prefix_lengths) - )) - - self.prefix_len = prefix_lengths.pop() - - expected_prefixes = set( - ''.join(letters) - for letters in product( - set(string.hexdigits.lower()), repeat=self.prefix_len - ) - ) - missing_prefixes = expected_prefixes - set(accounts) - if missing_prefixes: - raise ValueError("Missing prefixes %s" - % ', '.join(sorted(missing_prefixes))) - - self.prefixes = {} - request_session = requests.Session() - for prefix, account in accounts.items(): - self.prefixes[prefix] = ( - BlockBlobService( - account_name=account['account_name'], - account_key=account['api_secret_key'], - request_session=request_session, - ), - account['container_name'], - ) - - def get_blob_service(self, hex_obj_id): - """Get the block_blob_service and container that contains the object with - internal id hex_obj_id - """ - return self.prefixes[hex_obj_id[:self.prefix_len]] - - def get_all_blob_services(self): - """Get all active block_blob_services""" - yield from (v for k, v in sorted(self.prefixes.items())) Index: swh/objstorage/cloud/objstorage_cloud.py =================================================================== --- /dev/null +++ swh/objstorage/cloud/objstorage_cloud.py @@ -1,205 +0,0 @@ -# Copyright (C) 2016-2017 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import abc -import collections - -from swh.model import hashutil -from swh.objstorage.objstorage import ObjStorage, compute_hash -from swh.objstorage.objstorage import compressors, decompressors -from swh.objstorage.exc import ObjNotFoundError, Error - -from libcloud.storage import providers -from libcloud.storage.types import Provider, ObjectDoesNotExistError - - -class CloudObjStorage(ObjStorage, metaclass=abc.ABCMeta): - """Abstract ObjStorage that connect to a cloud using Libcloud - - Implementations of this class must redefine the _get_provider - method to make it return a driver provider (i.e. object that - supports `get_driver` method) which return a LibCloud driver (see - https://libcloud.readthedocs.io/en/latest/storage/api.html). - - """ - def __init__(self, container_name, compression=None, **kwargs): - super().__init__(**kwargs) - self.driver = self._get_driver(**kwargs) - self.container_name = container_name - self.container = self.driver.get_container( - container_name=container_name) - self.compression = compression - - def _get_driver(self, **kwargs): - """Initialize a driver to communicate with the cloud - - Kwargs: arguments passed to the StorageDriver class, typically - key: key to connect to the API. - secret: secret key for authentication. - secure: (bool) support HTTPS - host: (str) - port: (int) - api_version: (str) - region: (str) - - Returns: - a Libcloud driver to a cloud storage. - - """ - # Get the driver class from its description. - cls = providers.get_driver(self._get_provider()) - cls.namespace = None - # Initialize the driver. - return cls(**kwargs) - - @abc.abstractmethod - def _get_provider(self): - """Get a libcloud driver provider - - This method must be overridden by subclasses to specify which - of the native libcloud driver the current storage should - connect to. Alternatively, provider for a custom driver may - be returned, in which case the provider will have to support - `get_driver` method. - - """ - raise NotImplementedError('%s must implement `get_provider` method' - % type(self)) - - def check_config(self, *, check_write): - """Check the configuration for this object storage""" - # FIXME: hopefully this blew up during instantiation - return True - - def __contains__(self, obj_id): - try: - self._get_object(obj_id) - except ObjNotFoundError: - return False - else: - return True - - def __iter__(self): - """ Iterate over the objects present in the storage - - Warning: Iteration over the contents of a cloud-based object storage - may have bad efficiency: due to the very high amount of objects in it - and the fact that it is remote, get all the contents of the current - object storage may result in a lot of network requests. - - You almost certainly don't want to use this method in production. - """ - yield from (hashutil.bytehex_to_hash(obj.name.encode()) for obj in - self.driver.iterate_container_objects(self.container)) - - def __len__(self): - """Compute the number of objects in the current object storage. - - Warning: this currently uses `__iter__`, its warning about bad - performance applies. - - Returns: - number of objects contained in the storage. - - """ - return sum(1 for i in self) - - def add(self, content, obj_id=None, check_presence=True): - if obj_id is None: - # Checksum is missing, compute it on the fly. - obj_id = compute_hash(content) - - if check_presence and obj_id in self: - return obj_id - - self._put_object(content, obj_id) - return obj_id - - def restore(self, content, obj_id=None): - return self.add(content, obj_id, check_presence=False) - - def get(self, obj_id): - obj = b''.join(self._get_object(obj_id).as_stream()) - try: - return decompressors[self.compression](obj) - except Exception: - return obj - - def check(self, obj_id): - # Check that the file exists, as _get_object raises ObjNotFoundError - self._get_object(obj_id) - # Check the content integrity - obj_content = self.get(obj_id) - content_obj_id = compute_hash(obj_content) - if content_obj_id != obj_id: - raise Error(obj_id) - - def delete(self, obj_id): - super().delete(obj_id) # Check delete permission - obj = self._get_object(obj_id) - return self.driver.delete_object(obj) - - def _get_object(self, obj_id): - """Get a Libcloud wrapper for an object pointer. - - This wrapper does not retrieve the content of the object - directly. - - """ - hex_obj_id = hashutil.hash_to_hex(obj_id) - - try: - return self.driver.get_object(self.container_name, hex_obj_id) - except ObjectDoesNotExistError: - raise ObjNotFoundError(obj_id) - - def _compressor(self, data): - comp = compressors[self.compression]() - got_cchunk = False - for chunk in data: - cchunk = comp.compress(chunk) - if cchunk: - got_cchunk = True - yield cchunk - trail = comp.flush() - if got_cchunk: - yield trail - else: - data = decompressors[self.compression](trail) - if len(data) < len(trail): - yield data - else: - yield trail - - def _put_object(self, content, obj_id): - """Create an object in the cloud storage. - - Created object will contain the content and be referenced by - the given id. - - """ - hex_obj_id = hashutil.hash_to_hex(obj_id) - - if not isinstance(content, collections.Iterator): - content = (content,) - self.driver.upload_object_via_stream( - self._compressor(content), - self.container, hex_obj_id) - - -class AwsCloudObjStorage(CloudObjStorage): - """ Amazon's S3 Cloud-based object storage - - """ - def _get_provider(self): - return Provider.S3 - - -class OpenStackCloudObjStorage(CloudObjStorage): - """ OpenStack Swift Cloud based object storage - - """ - def _get_provider(self): - return Provider.OPENSTACK_SWIFT Index: swh/objstorage/objstorage_in_memory.py =================================================================== --- /dev/null +++ swh/objstorage/objstorage_in_memory.py @@ -1,72 +0,0 @@ -# Copyright (C) 2017 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import functools -import io - -from swh.objstorage.exc import ObjNotFoundError, Error -from swh.objstorage.objstorage import ObjStorage, compute_hash, \ - DEFAULT_CHUNK_SIZE - - -class InMemoryObjStorage(ObjStorage): - """In-Memory objstorage. - - Intended for test purposes. - - """ - - def __init__(self, **args): - super().__init__() - self.state = {} - - def check_config(self, *, check_write): - return True - - def __contains__(self, obj_id, *args, **kwargs): - return obj_id in self.state - - def __iter__(self): - return iter(sorted(self.state)) - - def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): - if obj_id is None: - obj_id = compute_hash(content) - - if check_presence and obj_id in self: - return obj_id - - self.state[obj_id] = content - - return obj_id - - def get(self, obj_id, *args, **kwargs): - if obj_id not in self: - raise ObjNotFoundError(obj_id) - - return self.state[obj_id] - - def check(self, obj_id, *args, **kwargs): - if obj_id not in self: - raise ObjNotFoundError(obj_id) - if compute_hash(self.state[obj_id]) != obj_id: - raise Error('Corrupt object %s' % obj_id) - return True - - def delete(self, obj_id, *args, **kwargs): - super().delete(obj_id) # Check delete permission - if obj_id not in self: - raise ObjNotFoundError(obj_id) - - self.state.pop(obj_id) - return True - - def get_stream(self, obj_id, chunk_size=DEFAULT_CHUNK_SIZE): - if obj_id not in self: - raise ObjNotFoundError(obj_id) - - data = io.BytesIO(self.state[obj_id]) - reader = functools.partial(data.read, chunk_size) - yield from iter(reader, b'') Index: swh/objstorage/objstorage_rados.py =================================================================== --- /dev/null +++ swh/objstorage/objstorage_rados.py @@ -1,91 +0,0 @@ -# Copyright (C) 2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import rados - -from swh.model import hashutil - -from swh.objstorage.exc import ObjNotFoundError -from swh.objstorage import objstorage - -READ_SIZE = 8192 - - -class RADOSObjStorage(objstorage.ObjStorage): - """Object storage implemented with RADOS""" - - def __init__(self, *, rados_id, pool_name, ceph_config, - allow_delete=False): - super().__init__(allow_delete=allow_delete) - self.pool_name = pool_name - self.cluster = rados.Rados( - conf=ceph_config, - conffile='', - rados_id=rados_id, - ) - self.cluster.connect() - self.__ioctx = None - - def check_config(self, *, check_write): - if self.pool_name not in self.cluster.list_pools(): - raise ValueError('Pool %s does not exist' % self.pool_name) - - @staticmethod - def _to_rados_obj_id(obj_id): - """Convert to a RADOS object identifier""" - return hashutil.hash_to_hex(obj_id) - - @property - def ioctx(self): - if not self.__ioctx: - self.__ioctx = self.cluster.open_ioctx(self.pool_name) - return self.__ioctx - - def __contains__(self, obj_id): - try: - self.ioctx.stat(self._to_rados_obj_id(obj_id)) - except rados.ObjectNotFound: - return False - else: - return True - - def add(self, content, obj_id=None, check_presence=True): - if not obj_id: - raise ValueError('add needs an obj_id') - - _obj_id = self._to_rados_obj_id(obj_id) - - if check_presence: - try: - self.ioctx.stat(_obj_id) - except rados.ObjectNotFound: - pass - else: - return obj_id - self.ioctx.write_full(_obj_id, content) - - return obj_id - - def get(self, obj_id): - chunks = [] - _obj_id = self._to_rados_obj_id(obj_id) - try: - length, mtime = self.ioctx.stat(_obj_id) - except rados.ObjectNotFound: - raise ObjNotFoundError(obj_id) from None - offset = 0 - while offset < length: - chunk = self.ioctx.read(_obj_id, offset, READ_SIZE) - chunks.append(chunk) - offset += len(chunk) - - return b''.join(chunks) - - def check(self, obj_id): - return True - - def delete(self, obj_id): - super().delete(obj_id) # check delete permission - return True Index: swh/objstorage/objstorage_weed.py =================================================================== --- /dev/null +++ swh/objstorage/objstorage_weed.py @@ -1,197 +0,0 @@ -# Copyright (C) 2019 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import os -import io -import logging -from urllib.parse import urljoin - -import requests - -from swh.model import hashutil -from swh.objstorage.objstorage import ObjStorage, compute_hash -from swh.objstorage.objstorage import compressors, decompressors - -from swh.objstorage.exc import ObjNotFoundError, Error - -LOGGER = logging.getLogger(__name__) -LOGGER.setLevel(logging.ERROR) - -DEFAULT_LIMIT = 1000 - - -class WeedFiler(object): - """ weed filer service. - """ - - def __init__(self, url): - self.url = url - - def get(self, remote_path): - url = urljoin(self.url, remote_path) - LOGGER.debug('Get file %s', url) - return requests.get(url).content - - def exists(self, remote_path): - url = urljoin(self.url, remote_path) - LOGGER.debug('Check file %s', url) - return requests.post(url).status_code == 200 - - def put(self, fp, remote_path): - url = urljoin(self.url, remote_path) - LOGGER.debug('Put file %s', url) - return requests.post(url, files={'file': fp}) - - def delete(self, remote_path): - url = urljoin(self.url, remote_path) - LOGGER.debug('Delete file %s', url) - return requests.delete(url) - - def list(self, dir, last_file_name=None, limit=DEFAULT_LIMIT): - '''list sub folders and files of @dir. show a better look if you turn on - - returns a dict of "sub-folders and files' - - ''' - d = dir if dir.endswith('/') else (dir + '/') - url = urljoin(self.url, d) - headers = {'Accept': 'application/json'} - params = {'limit': limit} - if last_file_name: - params['lastFileName'] = last_file_name - - LOGGER.debug('List directory %s', url) - rsp = requests.get(url, params=params, headers=headers) - if rsp.ok: - return rsp.json() - else: - LOGGER.error('Error listing "%s". [HTTP %d]' % ( - url, rsp.status_code)) - - -class WeedObjStorage(ObjStorage): - """ - """ - def __init__(self, url='http://127.0.0.1:8888/swh', - compression=None, **kwargs): - super().__init__(**kwargs) - self.wf = WeedFiler(url) - self.root_path = '/swh' - self.compression = compression - - def check_config(self, *, check_write): - """Check the configuration for this object storage""" - # FIXME: hopefully this blew up during instantiation - return True - - def __contains__(self, obj_id): - return self.wf.exists(obj_id) - - def __iter__(self): - """ Iterate over the objects present in the storage - - Warning: Iteration over the contents of a cloud-based object storage - may have bad efficiency: due to the very high amount of objects in it - and the fact that it is remote, get all the contents of the current - object storage may result in a lot of network requests. - - You almost certainly don't want to use this method in production. - """ - last_obj_id = None - while True: - for obj_id in self.list_content(last_obj_id=last_obj_id): - yield obj_id - if last_obj_id == obj_id: - break - last_obj_id = obj_id - - def __len__(self): - """Compute the number of objects in the current object storage. - - Warning: this currently uses `__iter__`, its warning about bad - performance applies. - - Returns: - number of objects contained in the storage. - - """ - return sum(1 for i in self) - - def add(self, content, obj_id=None, check_presence=True): - if obj_id is None: - # Checksum is missing, compute it on the fly. - obj_id = compute_hash(content) - - if check_presence and obj_id in self: - return obj_id - - self._put_object(content, obj_id) - return obj_id - - def restore(self, content, obj_id=None): - return self.add(content, obj_id, check_presence=False) - - def get(self, obj_id): - return self._get_object(obj_id) - - def check(self, obj_id): - # Check that the file exists, as _get_object raises ObjNotFoundError - self._get_object(obj_id) - # Check the content integrity - obj_content = self.get(obj_id) - content_obj_id = compute_hash(obj_content) - if content_obj_id != obj_id: - raise Error(obj_id) - - def delete(self, obj_id): - super().delete(obj_id) # Check delete permission - self.wf.delete(self.path(obj_id)) - - def _get_object(self, obj_id): - """Get a Libcloud wrapper for an object pointer. - - This wrapper does not retrieve the content of the object - directly. - - """ - try: - obj = self.wf.get(self.path(obj_id)) - return decompressors[self.compression](obj) - except Exception: - raise ObjNotFoundError(obj_id) - - def _put_object(self, content, obj_id): - """Create an object in the cloud storage. - - Created object will contain the content and be referenced by - the given id. - - """ - def compressor(data): - comp = compressors[self.compression]() - for chunk in data: - yield comp.compress(chunk) - yield comp.flush() - - if isinstance(content, bytes): - content = [content] - self.wf.put(io.BytesIO(b''.join(compressor(content))), - self.path(obj_id)) - - def path(self, obj_id): - hex_obj_id = hashutil.hash_to_hex(obj_id) - return os.path.join(self.root_path, hex_obj_id) - - def list_content(self, last_obj_id=None, limit=DEFAULT_LIMIT): - if last_obj_id: - last_obj_id = hashutil.hash_to_hex(last_obj_id) - resp = self.wf.list(self.root_path, last_obj_id, limit) - if resp is not None: - entries = resp['Entries'] - if entries: - for obj in entries: - if obj is not None: - bytehex = obj['FullPath'].rsplit('/', 1)[-1] - yield hashutil.bytehex_to_hash(bytehex.encode()) Index: swh/objstorage/tests/test_objstorage_azure.py =================================================================== --- swh/objstorage/tests/test_objstorage_azure.py +++ swh/objstorage/tests/test_objstorage_azure.py @@ -68,7 +68,7 @@ def setUp(self): super().setUp() patcher = patch( - 'swh.objstorage.cloud.objstorage_azure.BlockBlobService', + 'swh.objstorage.backends.azure.BlockBlobService', MockBlockBlobService, ) patcher.start() @@ -86,7 +86,7 @@ def setUp(self): super().setUp() patcher = patch( - 'swh.objstorage.cloud.objstorage_azure.BlockBlobService', + 'swh.objstorage.backends.azure.BlockBlobService', MockBlockBlobService, ) patcher.start()