diff --git a/swh/objstorage/factory.py b/swh/objstorage/factory.py --- a/swh/objstorage/factory.py +++ b/swh/objstorage/factory.py @@ -1,18 +1,11 @@ -# Copyright (C) 2016-2021 The Software Heritage developers +# Copyright (C) 2016-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from typing import Callable, Dict, Union +import importlib import warnings -from swh.objstorage.api.client import RemoteObjStorage -from swh.objstorage.backends.generator import RandomGeneratorObjStorage -from swh.objstorage.backends.http import HTTPReadOnlyObjStorage -from swh.objstorage.backends.in_memory import InMemoryObjStorage -from swh.objstorage.backends.noop import NoopObjStorage -from swh.objstorage.backends.pathslicing import PathSlicingObjStorage -from swh.objstorage.backends.seaweedfs import SeaweedFilerObjStorage from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage from swh.objstorage.multiplexer.filter import add_filters from swh.objstorage.objstorage import ObjStorage @@ -20,49 +13,21 @@ __all__ = ["get_objstorage", "ObjStorage"] -_STORAGE_CLASSES: Dict[str, Union[type, Callable[..., type]]] = { - "pathslicing": PathSlicingObjStorage, - "remote": RemoteObjStorage, - "memory": InMemoryObjStorage, - "seaweedfs": SeaweedFilerObjStorage, - "random": RandomGeneratorObjStorage, - "http": HTTPReadOnlyObjStorage, - "noop": NoopObjStorage, +OBJSTORAGE_IMPLEMENTATIONS = { + "pathslicing": ".backends.pathslicing.PathSlicingObjStorage", + "remote": ".api.client.RemoteObjStorage", + "memory": ".backends.in_memory.InMemoryObjStorage", + "seaweedfs": ".backends.seaweedfs.SeaweedFilerObjStorage", + "random": ".backends.generator.RandomGeneratorObjStorage", + "http": ".backends.http.HTTPReadOnlyObjStorage", + "noop": ".backends.noop.NoopObjStorage", + "azure": ".backends.azure.AzureCloudObjStorage", + "azure-prefixed": ".backends.azure.PrefixedAzureCloudObjStorage", + "s3": ".backends.libcloud.AwsCloudObjStorage", + "swift": ".backends.libcloud.OpenStackCloudObjStorage", + "winery": ".backends.winery.WineryObjStorage", } -_STORAGE_CLASSES_MISSING = {} - -try: - from swh.objstorage.backends.azure import ( - AzureCloudObjStorage, - PrefixedAzureCloudObjStorage, - ) - - _STORAGE_CLASSES["azure"] = AzureCloudObjStorage - _STORAGE_CLASSES["azure-prefixed"] = PrefixedAzureCloudObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["azure"] = e.args[0] - _STORAGE_CLASSES_MISSING["azure-prefixed"] = e.args[0] - -try: - from swh.objstorage.backends.libcloud import ( - AwsCloudObjStorage, - OpenStackCloudObjStorage, - ) - - _STORAGE_CLASSES["s3"] = AwsCloudObjStorage - _STORAGE_CLASSES["swift"] = OpenStackCloudObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["s3"] = e.args[0] - _STORAGE_CLASSES_MISSING["swift"] = e.args[0] - -try: - from swh.objstorage.backends.winery import WineryObjStorage - - _STORAGE_CLASSES["winery"] = WineryObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["winery"] = e.args[0] - def get_objstorage(cls: str, args=None, **kwargs): """Create an ObjStorage using the given implementation class. @@ -79,30 +44,41 @@ ValueError: if the given storage class is not a valid objstorage key. """ - if cls in _STORAGE_CLASSES: - if args is not None: - warnings.warn( - 'Explicit "args" key is deprecated for objstorage initialization, ' - "use class arguments keys directly instead.", - DeprecationWarning, - ) - # TODO: when removing this, drop the "args" backwards compatibility - # from swh.objstorage.api.server configuration checker - kwargs = args - return _STORAGE_CLASSES[cls](**kwargs) - else: + if args is not None: + warnings.warn( + 'Explicit "args" key is deprecated for objstorage initialization, ' + "use class arguments keys directly instead.", + DeprecationWarning, + ) + # TODO: when removing this, drop the "args" backwards compatibility + # from swh.objstorage.api.server configuration checker + kwargs = args + + class_path = OBJSTORAGE_IMPLEMENTATIONS.get(cls) + if class_path is None: raise ValueError( - "Storage class {} is not available: {}".format( - cls, _STORAGE_CLASSES_MISSING.get(cls, "unknown name") - ) + "Unknown storage class `%s`. Supported: %s" + % (cls, ", ".join(OBJSTORAGE_IMPLEMENTATIONS)) ) + if "." in class_path: + (module_path, class_name) = class_path.rsplit(".", 1) + try: + module = importlib.import_module(module_path, package=__package__) + except ImportError as e: + raise ValueError(f"Storage class {cls} is not available: {e.args[0]}") + ObjStorage = getattr(module, class_name) + else: + ObjStorage = globals()[class_path] + + return ObjStorage(**kwargs) + def _construct_filtered_objstorage(storage_conf, filters_conf): return add_filters(get_objstorage(**storage_conf), filters_conf) -_STORAGE_CLASSES["filtered"] = _construct_filtered_objstorage +OBJSTORAGE_IMPLEMENTATIONS["filtered"] = "_construct_filtered_objstorage" def _construct_multiplexer_objstorage(objstorages): @@ -110,7 +86,7 @@ return MultiplexerObjStorage(storages) -_STORAGE_CLASSES["multiplexer"] = _construct_multiplexer_objstorage +OBJSTORAGE_IMPLEMENTATIONS["multiplexer"] = "_construct_multiplexer_objstorage" def _construct_striping_objstorage(objstorages): @@ -118,4 +94,4 @@ return StripingObjStorage(storages) -_STORAGE_CLASSES["striping"] = _construct_striping_objstorage +OBJSTORAGE_IMPLEMENTATIONS["striping"] = "_construct_striping_objstorage"