Page MenuHomeSoftware Heritage

D3062.id10915.diff
No OneTemporary

D3062.id10915.diff

diff --git a/swh/objstorage/__init__.py b/swh/objstorage/__init__.py
--- a/swh/objstorage/__init__.py
+++ b/swh/objstorage/__init__.py
@@ -1,107 +1,12 @@
-# Copyright (C) 2016 The Software Heritage developers
+# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa
-from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
-from swh.objstorage.backends.in_memory import InMemoryObjStorage
-from swh.objstorage.api.client import RemoteObjStorage
-from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage
-from swh.objstorage.multiplexer.filter import add_filters
-from swh.objstorage.backends.seaweed import WeedObjStorage
-from swh.objstorage.backends.generator import RandomGeneratorObjStorage
+from typing import Iterable
+from pkgutil import extend_path
-from typing import Callable, Dict, Union
+__path__: Iterable[str] = extend_path(__path__, __name__)
-__all__ = ["get_objstorage", "ObjStorage"]
-
-
-_STORAGE_CLASSES: Dict[str, Union[type, Callable[..., type]]] = {
- "pathslicing": PathSlicingObjStorage,
- "remote": RemoteObjStorage,
- "memory": InMemoryObjStorage,
- "weed": WeedObjStorage,
- "random": RandomGeneratorObjStorage,
-}
-
-_STORAGE_CLASSES_MISSING = {}
-
-try:
- from swh.objstorage.backends.azure import (
- AzureCloudObjStorage,
- PrefixedAzureCloudObjStorage,
- )
-
- _STORAGE_CLASSES["azure"] = AzureCloudObjStorage
- _STORAGE_CLASSES["azure-prefixed"] = PrefixedAzureCloudObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["azure"] = e.args[0]
- _STORAGE_CLASSES_MISSING["azure-prefixed"] = e.args[0]
-
-try:
- from swh.objstorage.backends.rados import RADOSObjStorage
-
- _STORAGE_CLASSES["rados"] = RADOSObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["rados"] = e.args[0]
-
-try:
- from swh.objstorage.backends.libcloud import (
- AwsCloudObjStorage,
- OpenStackCloudObjStorage,
- )
-
- _STORAGE_CLASSES["s3"] = AwsCloudObjStorage
- _STORAGE_CLASSES["swift"] = OpenStackCloudObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["s3"] = e.args[0]
- _STORAGE_CLASSES_MISSING["swift"] = e.args[0]
-
-
-def get_objstorage(cls, args):
- """ Create an ObjStorage using the given implementation class.
-
- Args:
- cls (str): objstorage class unique key contained in the
- _STORAGE_CLASSES dict.
- args (dict): arguments for the required class of objstorage
- that must match exactly the one in the `__init__` method of the
- class.
- Returns:
- subclass of ObjStorage that match the given `storage_class` argument.
- Raises:
- ValueError: if the given storage class is not a valid objstorage
- key.
- """
- if cls in _STORAGE_CLASSES:
- return _STORAGE_CLASSES[cls](**args)
- else:
- raise ValueError(
- "Storage class {} is not available: {}".format(
- cls, _STORAGE_CLASSES_MISSING.get(cls, "unknown name")
- )
- )
-
-
-def _construct_filtered_objstorage(storage_conf, filters_conf):
- return add_filters(get_objstorage(**storage_conf), filters_conf)
-
-
-_STORAGE_CLASSES["filtered"] = _construct_filtered_objstorage
-
-
-def _construct_multiplexer_objstorage(objstorages):
- storages = [get_objstorage(**conf) for conf in objstorages]
- return MultiplexerObjStorage(storages)
-
-
-_STORAGE_CLASSES["multiplexer"] = _construct_multiplexer_objstorage
-
-
-def _construct_striping_objstorage(objstorages):
- storages = [get_objstorage(**conf) for conf in objstorages]
- return StripingObjStorage(storages)
-
-
-_STORAGE_CLASSES["striping"] = _construct_striping_objstorage
+# for BW compat
+from swh.objstorage.factory import * # noqa
diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py
--- a/swh/objstorage/api/server.py
+++ b/swh/objstorage/api/server.py
@@ -19,7 +19,7 @@
from swh.core.api.serializers import msgpack_loads, SWHJSONDecoder
from swh.model import hashutil
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.objstorage import DEFAULT_LIMIT
from swh.objstorage.exc import Error, ObjNotFoundError
from swh.core.statsd import statsd
diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py
--- a/swh/objstorage/cli.py
+++ b/swh/objstorage/cli.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -12,8 +12,8 @@
from swh.core.cli import CONTEXT_SETTINGS
-from swh.objstorage import get_objstorage
from swh.objstorage.api.server import load_and_check_config, make_app
+from swh.objstorage.factory import get_objstorage
@click.group(name="objstorage", context_settings=CONTEXT_SETTINGS)
diff --git a/swh/objstorage/__init__.py b/swh/objstorage/factory.py
copy from swh/objstorage/__init__.py
copy to swh/objstorage/factory.py
--- a/swh/objstorage/__init__.py
+++ b/swh/objstorage/factory.py
@@ -1,8 +1,10 @@
-# Copyright (C) 2016 The Software Heritage developers
+# Copyright (C) 2016-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+from typing import Callable, Dict, Union
+
from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa
from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
from swh.objstorage.backends.in_memory import InMemoryObjStorage
@@ -12,7 +14,6 @@
from swh.objstorage.backends.seaweed import WeedObjStorage
from swh.objstorage.backends.generator import RandomGeneratorObjStorage
-from typing import Callable, Dict, Union
__all__ = ["get_objstorage", "ObjStorage"]
diff --git a/swh/objstorage/tests/test_multiplexer_filter.py b/swh/objstorage/tests/test_multiplexer_filter.py
--- a/swh/objstorage/tests/test_multiplexer_filter.py
+++ b/swh/objstorage/tests/test_multiplexer_filter.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -10,7 +10,7 @@
from string import ascii_lowercase
from swh.model import hashutil
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.exc import Error, ObjNotFoundError
from swh.objstorage.multiplexer.filter import id_prefix, id_regex, read_only
from swh.objstorage.objstorage import compute_hash
diff --git a/swh/objstorage/tests/test_objstorage_api.py b/swh/objstorage/tests/test_objstorage_api.py
--- a/swh/objstorage/tests/test_objstorage_api.py
+++ b/swh/objstorage/tests/test_objstorage_api.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,7 +9,7 @@
import pytest
from swh.core.api.tests.server_testing import ServerTestFixtureAsync
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.api.server import make_app
from swh.objstorage.tests.objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_azure.py b/swh/objstorage/tests/test_objstorage_azure.py
--- a/swh/objstorage/tests/test_objstorage_azure.py
+++ b/swh/objstorage/tests/test_objstorage_azure.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 The Software Heritage developers
+# Copyright (C) 2016-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -12,7 +12,7 @@
from azure.common import AzureMissingResourceHttpError
from swh.model.hashutil import hash_to_hex
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.objstorage import decompressors
from swh.objstorage.exc import Error
diff --git a/swh/objstorage/tests/test_objstorage_in_memory.py b/swh/objstorage/tests/test_objstorage_in_memory.py
--- a/swh/objstorage/tests/test_objstorage_in_memory.py
+++ b/swh/objstorage/tests/test_objstorage_in_memory.py
@@ -1,11 +1,11 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from .objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_instantiation.py b/swh/objstorage/tests/test_objstorage_instantiation.py
--- a/swh/objstorage/tests/test_objstorage_instantiation.py
+++ b/swh/objstorage/tests/test_objstorage_instantiation.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -7,7 +7,7 @@
import tempfile
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.api.client import RemoteObjStorage
from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
diff --git a/swh/objstorage/tests/test_objstorage_multiplexer.py b/swh/objstorage/tests/test_objstorage_multiplexer.py
--- a/swh/objstorage/tests/test_objstorage_multiplexer.py
+++ b/swh/objstorage/tests/test_objstorage_multiplexer.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -8,7 +8,7 @@
import tempfile
import unittest
-from swh.objstorage import PathSlicingObjStorage
+from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
from swh.objstorage.multiplexer import MultiplexerObjStorage
from swh.objstorage.multiplexer.filter import add_filter, read_only
diff --git a/swh/objstorage/tests/test_objstorage_pathslicing.py b/swh/objstorage/tests/test_objstorage_pathslicing.py
--- a/swh/objstorage/tests/test_objstorage_pathslicing.py
+++ b/swh/objstorage/tests/test_objstorage_pathslicing.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,7 +9,9 @@
from unittest.mock import patch, DEFAULT
from swh.model import hashutil
-from swh.objstorage import exc, get_objstorage, ID_HASH_LENGTH
+from swh.objstorage import exc
+from swh.objstorage.factory import get_objstorage
+from swh.objstorage.objstorage import ID_HASH_LENGTH
from .objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_random_generator.py b/swh/objstorage/tests/test_objstorage_random_generator.py
--- a/swh/objstorage/tests/test_objstorage_random_generator.py
+++ b/swh/objstorage/tests/test_objstorage_random_generator.py
@@ -1,10 +1,10 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from collections import Iterator
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
def test_random_generator_objstorage():
diff --git a/swh/objstorage/tests/test_objstorage_striping.py b/swh/objstorage/tests/test_objstorage_striping.py
--- a/swh/objstorage/tests/test_objstorage_striping.py
+++ b/swh/objstorage/tests/test_objstorage_striping.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -8,7 +8,7 @@
import tempfile
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from .objstorage_testing import ObjStorageTestFixture

File Metadata

Mime Type
text/plain
Expires
Jan 30 2025, 12:06 PM (6 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220094

Event Timeline