diff --git a/swh/objstorage/__init__.py b/swh/objstorage/__init__.py --- a/swh/objstorage/__init__.py +++ b/swh/objstorage/__init__.py @@ -1,107 +1,12 @@ -# Copyright (C) 2016 The Software Heritage developers +# Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa -from swh.objstorage.backends.pathslicing import PathSlicingObjStorage -from swh.objstorage.backends.in_memory import InMemoryObjStorage -from swh.objstorage.api.client import RemoteObjStorage -from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage -from swh.objstorage.multiplexer.filter import add_filters -from swh.objstorage.backends.seaweed import WeedObjStorage -from swh.objstorage.backends.generator import RandomGeneratorObjStorage +from typing import Iterable +from pkgutil import extend_path -from typing import Callable, Dict, Union +__path__: Iterable[str] = extend_path(__path__, __name__) -__all__ = ["get_objstorage", "ObjStorage"] - - -_STORAGE_CLASSES: Dict[str, Union[type, Callable[..., type]]] = { - "pathslicing": PathSlicingObjStorage, - "remote": RemoteObjStorage, - "memory": InMemoryObjStorage, - "weed": WeedObjStorage, - "random": RandomGeneratorObjStorage, -} - -_STORAGE_CLASSES_MISSING = {} - -try: - from swh.objstorage.backends.azure import ( - AzureCloudObjStorage, - PrefixedAzureCloudObjStorage, - ) - - _STORAGE_CLASSES["azure"] = AzureCloudObjStorage - _STORAGE_CLASSES["azure-prefixed"] = PrefixedAzureCloudObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["azure"] = e.args[0] - _STORAGE_CLASSES_MISSING["azure-prefixed"] = e.args[0] - -try: - from swh.objstorage.backends.rados import RADOSObjStorage - - _STORAGE_CLASSES["rados"] = RADOSObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["rados"] = e.args[0] - -try: - from swh.objstorage.backends.libcloud import ( - AwsCloudObjStorage, - OpenStackCloudObjStorage, - ) - - _STORAGE_CLASSES["s3"] = AwsCloudObjStorage - _STORAGE_CLASSES["swift"] = OpenStackCloudObjStorage -except ImportError as e: - _STORAGE_CLASSES_MISSING["s3"] = e.args[0] - _STORAGE_CLASSES_MISSING["swift"] = e.args[0] - - -def get_objstorage(cls, args): - """ Create an ObjStorage using the given implementation class. - - Args: - cls (str): objstorage class unique key contained in the - _STORAGE_CLASSES dict. - args (dict): arguments for the required class of objstorage - that must match exactly the one in the `__init__` method of the - class. - Returns: - subclass of ObjStorage that match the given `storage_class` argument. - Raises: - ValueError: if the given storage class is not a valid objstorage - key. - """ - if cls in _STORAGE_CLASSES: - return _STORAGE_CLASSES[cls](**args) - else: - raise ValueError( - "Storage class {} is not available: {}".format( - cls, _STORAGE_CLASSES_MISSING.get(cls, "unknown name") - ) - ) - - -def _construct_filtered_objstorage(storage_conf, filters_conf): - return add_filters(get_objstorage(**storage_conf), filters_conf) - - -_STORAGE_CLASSES["filtered"] = _construct_filtered_objstorage - - -def _construct_multiplexer_objstorage(objstorages): - storages = [get_objstorage(**conf) for conf in objstorages] - return MultiplexerObjStorage(storages) - - -_STORAGE_CLASSES["multiplexer"] = _construct_multiplexer_objstorage - - -def _construct_striping_objstorage(objstorages): - storages = [get_objstorage(**conf) for conf in objstorages] - return StripingObjStorage(storages) - - -_STORAGE_CLASSES["striping"] = _construct_striping_objstorage +# for BW compat +from swh.objstorage.factory import * # noqa diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py --- a/swh/objstorage/api/server.py +++ b/swh/objstorage/api/server.py @@ -19,7 +19,7 @@ from swh.core.api.serializers import msgpack_loads, SWHJSONDecoder from swh.model import hashutil -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from swh.objstorage.objstorage import DEFAULT_LIMIT from swh.objstorage.exc import Error, ObjNotFoundError from swh.core.statsd import statsd diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py --- a/swh/objstorage/cli.py +++ b/swh/objstorage/cli.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -12,8 +12,8 @@ from swh.core.cli import CONTEXT_SETTINGS -from swh.objstorage import get_objstorage from swh.objstorage.api.server import load_and_check_config, make_app +from swh.objstorage.factory import get_objstorage @click.group(name="objstorage", context_settings=CONTEXT_SETTINGS) diff --git a/swh/objstorage/__init__.py b/swh/objstorage/factory.py copy from swh/objstorage/__init__.py copy to swh/objstorage/factory.py --- a/swh/objstorage/__init__.py +++ b/swh/objstorage/factory.py @@ -1,8 +1,10 @@ -# Copyright (C) 2016 The Software Heritage developers +# Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from typing import Callable, Dict, Union + from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa from swh.objstorage.backends.pathslicing import PathSlicingObjStorage from swh.objstorage.backends.in_memory import InMemoryObjStorage @@ -12,7 +14,6 @@ from swh.objstorage.backends.seaweed import WeedObjStorage from swh.objstorage.backends.generator import RandomGeneratorObjStorage -from typing import Callable, Dict, Union __all__ = ["get_objstorage", "ObjStorage"] diff --git a/swh/objstorage/tests/test_multiplexer_filter.py b/swh/objstorage/tests/test_multiplexer_filter.py --- a/swh/objstorage/tests/test_multiplexer_filter.py +++ b/swh/objstorage/tests/test_multiplexer_filter.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -10,7 +10,7 @@ from string import ascii_lowercase from swh.model import hashutil -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from swh.objstorage.exc import Error, ObjNotFoundError from swh.objstorage.multiplexer.filter import id_prefix, id_regex, read_only from swh.objstorage.objstorage import compute_hash diff --git a/swh/objstorage/tests/test_objstorage_api.py b/swh/objstorage/tests/test_objstorage_api.py --- a/swh/objstorage/tests/test_objstorage_api.py +++ b/swh/objstorage/tests/test_objstorage_api.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2019 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,7 +9,7 @@ import pytest from swh.core.api.tests.server_testing import ServerTestFixtureAsync -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from swh.objstorage.api.server import make_app from swh.objstorage.tests.objstorage_testing import ObjStorageTestFixture diff --git a/swh/objstorage/tests/test_objstorage_azure.py b/swh/objstorage/tests/test_objstorage_azure.py --- a/swh/objstorage/tests/test_objstorage_azure.py +++ b/swh/objstorage/tests/test_objstorage_azure.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2018 The Software Heritage developers +# Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -12,7 +12,7 @@ from azure.common import AzureMissingResourceHttpError from swh.model.hashutil import hash_to_hex -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from swh.objstorage.objstorage import decompressors from swh.objstorage.exc import Error diff --git a/swh/objstorage/tests/test_objstorage_in_memory.py b/swh/objstorage/tests/test_objstorage_in_memory.py --- a/swh/objstorage/tests/test_objstorage_in_memory.py +++ b/swh/objstorage/tests/test_objstorage_in_memory.py @@ -1,11 +1,11 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import unittest -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from .objstorage_testing import ObjStorageTestFixture diff --git a/swh/objstorage/tests/test_objstorage_instantiation.py b/swh/objstorage/tests/test_objstorage_instantiation.py --- a/swh/objstorage/tests/test_objstorage_instantiation.py +++ b/swh/objstorage/tests/test_objstorage_instantiation.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -7,7 +7,7 @@ import tempfile import unittest -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from swh.objstorage.api.client import RemoteObjStorage from swh.objstorage.backends.pathslicing import PathSlicingObjStorage diff --git a/swh/objstorage/tests/test_objstorage_multiplexer.py b/swh/objstorage/tests/test_objstorage_multiplexer.py --- a/swh/objstorage/tests/test_objstorage_multiplexer.py +++ b/swh/objstorage/tests/test_objstorage_multiplexer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -8,7 +8,7 @@ import tempfile import unittest -from swh.objstorage import PathSlicingObjStorage +from swh.objstorage.backends.pathslicing import PathSlicingObjStorage from swh.objstorage.multiplexer import MultiplexerObjStorage from swh.objstorage.multiplexer.filter import add_filter, read_only diff --git a/swh/objstorage/tests/test_objstorage_pathslicing.py b/swh/objstorage/tests/test_objstorage_pathslicing.py --- a/swh/objstorage/tests/test_objstorage_pathslicing.py +++ b/swh/objstorage/tests/test_objstorage_pathslicing.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2017 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,7 +9,9 @@ from unittest.mock import patch, DEFAULT from swh.model import hashutil -from swh.objstorage import exc, get_objstorage, ID_HASH_LENGTH +from swh.objstorage import exc +from swh.objstorage.factory import get_objstorage +from swh.objstorage.objstorage import ID_HASH_LENGTH from .objstorage_testing import ObjStorageTestFixture diff --git a/swh/objstorage/tests/test_objstorage_random_generator.py b/swh/objstorage/tests/test_objstorage_random_generator.py --- a/swh/objstorage/tests/test_objstorage_random_generator.py +++ b/swh/objstorage/tests/test_objstorage_random_generator.py @@ -1,10 +1,10 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from collections import Iterator -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage def test_random_generator_objstorage(): diff --git a/swh/objstorage/tests/test_objstorage_striping.py b/swh/objstorage/tests/test_objstorage_striping.py --- a/swh/objstorage/tests/test_objstorage_striping.py +++ b/swh/objstorage/tests/test_objstorage_striping.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -8,7 +8,7 @@ import tempfile import unittest -from swh.objstorage import get_objstorage +from swh.objstorage.factory import get_objstorage from .objstorage_testing import ObjStorageTestFixture