diff --git a/mypy.ini b/mypy.ini index ac539ab..92ecb16 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,24 +1,21 @@ [mypy] namespace_packages = True warn_unused_ignores = True # 3rd party libraries without stubs (yet) [mypy-azure.*] ignore_missing_imports = True [mypy-libcloud.*] ignore_missing_imports = True [mypy-pkg_resources.*] ignore_missing_imports = True [mypy-pytest.*] ignore_missing_imports = True -[mypy-rados.*] -ignore_missing_imports = True - [mypy-requests_toolbelt.*] ignore_missing_imports = True diff --git a/requirements-test.txt b/requirements-test.txt index afd3efc..20c3661 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,8 +1,7 @@ apache-libcloud azure-storage-blob >= 12.0, != 12.9.0 # version 12.9.0 breaks mypy https://github.com/Azure/azure-sdk-for-python/pull/20891 pytest -python-cephlibs requests_mock[fixture] >= 1.9 requests_toolbelt types-pyyaml types-requests diff --git a/swh/objstorage/backends/rados.py b/swh/objstorage/backends/rados.py deleted file mode 100644 index 2bee06b..0000000 --- a/swh/objstorage/backends/rados.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (C) 2018 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import rados - -from swh.model import hashutil -from swh.objstorage import objstorage -from swh.objstorage.exc import ObjNotFoundError - -READ_SIZE = 8192 - - -class RADOSObjStorage(objstorage.ObjStorage): - """Object storage implemented with RADOS""" - - def __init__(self, *, rados_id, pool_name, ceph_config, allow_delete=False): - super().__init__(allow_delete=allow_delete) - self.pool_name = pool_name - self.cluster = rados.Rados(conf=ceph_config, conffile="", rados_id=rados_id,) - self.cluster.connect() - self.__ioctx = None - - def check_config(self, *, check_write): - if self.pool_name not in self.cluster.list_pools(): - raise ValueError("Pool %s does not exist" % self.pool_name) - - @staticmethod - def _to_rados_obj_id(obj_id): - """Convert to a RADOS object identifier""" - return hashutil.hash_to_hex(obj_id) - - @property - def ioctx(self): - if not self.__ioctx: - self.__ioctx = self.cluster.open_ioctx(self.pool_name) - return self.__ioctx - - def __contains__(self, obj_id): - try: - self.ioctx.stat(self._to_rados_obj_id(obj_id)) - except rados.ObjectNotFound: - return False - else: - return True - - def add(self, content, obj_id=None, check_presence=True): - if not obj_id: - raise ValueError("add needs an obj_id") - - _obj_id = self._to_rados_obj_id(obj_id) - - if check_presence: - try: - self.ioctx.stat(_obj_id) - except rados.ObjectNotFound: - pass - else: - return obj_id - self.ioctx.write_full(_obj_id, content) - - return obj_id - - def get(self, obj_id): - chunks = [] - _obj_id = self._to_rados_obj_id(obj_id) - try: - length, mtime = self.ioctx.stat(_obj_id) - except rados.ObjectNotFound: - raise ObjNotFoundError(obj_id) from None - offset = 0 - while offset < length: - chunk = self.ioctx.read(_obj_id, offset, READ_SIZE) - chunks.append(chunk) - offset += len(chunk) - - return b"".join(chunks) - - def check(self, obj_id): - return True - - def delete(self, obj_id): - super().delete(obj_id) # check delete permission - return True diff --git a/swh/objstorage/factory.py b/swh/objstorage/factory.py index 3f96784..d720397 100644 --- a/swh/objstorage/factory.py +++ b/swh/objstorage/factory.py @@ -1,129 +1,122 @@ # Copyright (C) 2016-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from typing import Callable, Dict, Union import warnings from swh.objstorage.api.client import RemoteObjStorage from swh.objstorage.backends.generator import RandomGeneratorObjStorage from swh.objstorage.backends.http import HTTPReadOnlyObjStorage from swh.objstorage.backends.in_memory import InMemoryObjStorage from swh.objstorage.backends.noop import NoopObjStorage from swh.objstorage.backends.pathslicing import PathSlicingObjStorage from swh.objstorage.backends.seaweedfs import SeaweedFilerObjStorage from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage from swh.objstorage.multiplexer.filter import add_filters from swh.objstorage.objstorage import ID_HASH_LENGTH, ObjStorage # noqa __all__ = ["get_objstorage", "ObjStorage"] _STORAGE_CLASSES: Dict[str, Union[type, Callable[..., type]]] = { "pathslicing": PathSlicingObjStorage, "remote": RemoteObjStorage, "memory": InMemoryObjStorage, "seaweedfs": SeaweedFilerObjStorage, "random": RandomGeneratorObjStorage, "http": HTTPReadOnlyObjStorage, "noop": NoopObjStorage, } _STORAGE_CLASSES_MISSING = {} _STORAGE_CLASSES_DEPRECATED = {"weed": "seaweedfs"} try: from swh.objstorage.backends.azure import ( AzureCloudObjStorage, PrefixedAzureCloudObjStorage, ) _STORAGE_CLASSES["azure"] = AzureCloudObjStorage _STORAGE_CLASSES["azure-prefixed"] = PrefixedAzureCloudObjStorage except ImportError as e: _STORAGE_CLASSES_MISSING["azure"] = e.args[0] _STORAGE_CLASSES_MISSING["azure-prefixed"] = e.args[0] -try: - from swh.objstorage.backends.rados import RADOSObjStorage - - _STORAGE_CLASSES["rados"] = RADOSObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["rados"] = e.args[0] - try: from swh.objstorage.backends.libcloud import ( AwsCloudObjStorage, OpenStackCloudObjStorage, ) _STORAGE_CLASSES["s3"] = AwsCloudObjStorage _STORAGE_CLASSES["swift"] = OpenStackCloudObjStorage except ImportError as e: _STORAGE_CLASSES_MISSING["s3"] = e.args[0] _STORAGE_CLASSES_MISSING["swift"] = e.args[0] def get_objstorage(cls: str, args=None, **kwargs): """ Create an ObjStorage using the given implementation class. Args: cls: objstorage class unique key contained in the _STORAGE_CLASSES dict. kwargs: arguments for the required class of objstorage that must match exactly the one in the `__init__` method of the class. Returns: subclass of ObjStorage that match the given `storage_class` argument. Raises: ValueError: if the given storage class is not a valid objstorage key. """ if cls in _STORAGE_CLASSES_DEPRECATED: warnings.warn( f"{cls} objstorage class is deprecated, " f"use {_STORAGE_CLASSES_DEPRECATED[cls]} class instead.", DeprecationWarning, ) cls = _STORAGE_CLASSES_DEPRECATED[cls] if cls in _STORAGE_CLASSES: if args is not None: warnings.warn( 'Explicit "args" key is deprecated for objstorage initialization, ' "use class arguments keys directly instead.", DeprecationWarning, ) # TODO: when removing this, drop the "args" backwards compatibility # from swh.objstorage.api.server configuration checker kwargs = args return _STORAGE_CLASSES[cls](**kwargs) else: raise ValueError( "Storage class {} is not available: {}".format( cls, _STORAGE_CLASSES_MISSING.get(cls, "unknown name") ) ) def _construct_filtered_objstorage(storage_conf, filters_conf): return add_filters(get_objstorage(**storage_conf), filters_conf) _STORAGE_CLASSES["filtered"] = _construct_filtered_objstorage def _construct_multiplexer_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return MultiplexerObjStorage(storages) _STORAGE_CLASSES["multiplexer"] = _construct_multiplexer_objstorage def _construct_striping_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return StripingObjStorage(storages) _STORAGE_CLASSES["striping"] = _construct_striping_objstorage