Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7163660
D3062.id10915.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
13 KB
Subscribers
None
D3062.id10915.diff
View Options
diff --git a/swh/objstorage/__init__.py b/swh/objstorage/__init__.py
--- a/swh/objstorage/__init__.py
+++ b/swh/objstorage/__init__.py
@@ -1,107 +1,12 @@
-# Copyright (C) 2016 The Software Heritage developers
+# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
-from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa
-from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
-from swh.objstorage.backends.in_memory import InMemoryObjStorage
-from swh.objstorage.api.client import RemoteObjStorage
-from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage
-from swh.objstorage.multiplexer.filter import add_filters
-from swh.objstorage.backends.seaweed import WeedObjStorage
-from swh.objstorage.backends.generator import RandomGeneratorObjStorage
+from typing import Iterable
+from pkgutil import extend_path
-from typing import Callable, Dict, Union
+__path__: Iterable[str] = extend_path(__path__, __name__)
-__all__ = ["get_objstorage", "ObjStorage"]
-
-
-_STORAGE_CLASSES: Dict[str, Union[type, Callable[..., type]]] = {
- "pathslicing": PathSlicingObjStorage,
- "remote": RemoteObjStorage,
- "memory": InMemoryObjStorage,
- "weed": WeedObjStorage,
- "random": RandomGeneratorObjStorage,
-}
-
-_STORAGE_CLASSES_MISSING = {}
-
-try:
- from swh.objstorage.backends.azure import (
- AzureCloudObjStorage,
- PrefixedAzureCloudObjStorage,
- )
-
- _STORAGE_CLASSES["azure"] = AzureCloudObjStorage
- _STORAGE_CLASSES["azure-prefixed"] = PrefixedAzureCloudObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["azure"] = e.args[0]
- _STORAGE_CLASSES_MISSING["azure-prefixed"] = e.args[0]
-
-try:
- from swh.objstorage.backends.rados import RADOSObjStorage
-
- _STORAGE_CLASSES["rados"] = RADOSObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["rados"] = e.args[0]
-
-try:
- from swh.objstorage.backends.libcloud import (
- AwsCloudObjStorage,
- OpenStackCloudObjStorage,
- )
-
- _STORAGE_CLASSES["s3"] = AwsCloudObjStorage
- _STORAGE_CLASSES["swift"] = OpenStackCloudObjStorage
-except ImportError as e:
- _STORAGE_CLASSES_MISSING["s3"] = e.args[0]
- _STORAGE_CLASSES_MISSING["swift"] = e.args[0]
-
-
-def get_objstorage(cls, args):
- """ Create an ObjStorage using the given implementation class.
-
- Args:
- cls (str): objstorage class unique key contained in the
- _STORAGE_CLASSES dict.
- args (dict): arguments for the required class of objstorage
- that must match exactly the one in the `__init__` method of the
- class.
- Returns:
- subclass of ObjStorage that match the given `storage_class` argument.
- Raises:
- ValueError: if the given storage class is not a valid objstorage
- key.
- """
- if cls in _STORAGE_CLASSES:
- return _STORAGE_CLASSES[cls](**args)
- else:
- raise ValueError(
- "Storage class {} is not available: {}".format(
- cls, _STORAGE_CLASSES_MISSING.get(cls, "unknown name")
- )
- )
-
-
-def _construct_filtered_objstorage(storage_conf, filters_conf):
- return add_filters(get_objstorage(**storage_conf), filters_conf)
-
-
-_STORAGE_CLASSES["filtered"] = _construct_filtered_objstorage
-
-
-def _construct_multiplexer_objstorage(objstorages):
- storages = [get_objstorage(**conf) for conf in objstorages]
- return MultiplexerObjStorage(storages)
-
-
-_STORAGE_CLASSES["multiplexer"] = _construct_multiplexer_objstorage
-
-
-def _construct_striping_objstorage(objstorages):
- storages = [get_objstorage(**conf) for conf in objstorages]
- return StripingObjStorage(storages)
-
-
-_STORAGE_CLASSES["striping"] = _construct_striping_objstorage
+# for BW compat
+from swh.objstorage.factory import * # noqa
diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py
--- a/swh/objstorage/api/server.py
+++ b/swh/objstorage/api/server.py
@@ -19,7 +19,7 @@
from swh.core.api.serializers import msgpack_loads, SWHJSONDecoder
from swh.model import hashutil
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.objstorage import DEFAULT_LIMIT
from swh.objstorage.exc import Error, ObjNotFoundError
from swh.core.statsd import statsd
diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py
--- a/swh/objstorage/cli.py
+++ b/swh/objstorage/cli.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -12,8 +12,8 @@
from swh.core.cli import CONTEXT_SETTINGS
-from swh.objstorage import get_objstorage
from swh.objstorage.api.server import load_and_check_config, make_app
+from swh.objstorage.factory import get_objstorage
@click.group(name="objstorage", context_settings=CONTEXT_SETTINGS)
diff --git a/swh/objstorage/__init__.py b/swh/objstorage/factory.py
copy from swh/objstorage/__init__.py
copy to swh/objstorage/factory.py
--- a/swh/objstorage/__init__.py
+++ b/swh/objstorage/factory.py
@@ -1,8 +1,10 @@
-# Copyright (C) 2016 The Software Heritage developers
+# Copyright (C) 2016-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+from typing import Callable, Dict, Union
+
from swh.objstorage.objstorage import ObjStorage, ID_HASH_LENGTH # noqa
from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
from swh.objstorage.backends.in_memory import InMemoryObjStorage
@@ -12,7 +14,6 @@
from swh.objstorage.backends.seaweed import WeedObjStorage
from swh.objstorage.backends.generator import RandomGeneratorObjStorage
-from typing import Callable, Dict, Union
__all__ = ["get_objstorage", "ObjStorage"]
diff --git a/swh/objstorage/tests/test_multiplexer_filter.py b/swh/objstorage/tests/test_multiplexer_filter.py
--- a/swh/objstorage/tests/test_multiplexer_filter.py
+++ b/swh/objstorage/tests/test_multiplexer_filter.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -10,7 +10,7 @@
from string import ascii_lowercase
from swh.model import hashutil
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.exc import Error, ObjNotFoundError
from swh.objstorage.multiplexer.filter import id_prefix, id_regex, read_only
from swh.objstorage.objstorage import compute_hash
diff --git a/swh/objstorage/tests/test_objstorage_api.py b/swh/objstorage/tests/test_objstorage_api.py
--- a/swh/objstorage/tests/test_objstorage_api.py
+++ b/swh/objstorage/tests/test_objstorage_api.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2019 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,7 +9,7 @@
import pytest
from swh.core.api.tests.server_testing import ServerTestFixtureAsync
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.api.server import make_app
from swh.objstorage.tests.objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_azure.py b/swh/objstorage/tests/test_objstorage_azure.py
--- a/swh/objstorage/tests/test_objstorage_azure.py
+++ b/swh/objstorage/tests/test_objstorage_azure.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2016-2018 The Software Heritage developers
+# Copyright (C) 2016-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -12,7 +12,7 @@
from azure.common import AzureMissingResourceHttpError
from swh.model.hashutil import hash_to_hex
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.objstorage import decompressors
from swh.objstorage.exc import Error
diff --git a/swh/objstorage/tests/test_objstorage_in_memory.py b/swh/objstorage/tests/test_objstorage_in_memory.py
--- a/swh/objstorage/tests/test_objstorage_in_memory.py
+++ b/swh/objstorage/tests/test_objstorage_in_memory.py
@@ -1,11 +1,11 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from .objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_instantiation.py b/swh/objstorage/tests/test_objstorage_instantiation.py
--- a/swh/objstorage/tests/test_objstorage_instantiation.py
+++ b/swh/objstorage/tests/test_objstorage_instantiation.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -7,7 +7,7 @@
import tempfile
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from swh.objstorage.api.client import RemoteObjStorage
from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
diff --git a/swh/objstorage/tests/test_objstorage_multiplexer.py b/swh/objstorage/tests/test_objstorage_multiplexer.py
--- a/swh/objstorage/tests/test_objstorage_multiplexer.py
+++ b/swh/objstorage/tests/test_objstorage_multiplexer.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -8,7 +8,7 @@
import tempfile
import unittest
-from swh.objstorage import PathSlicingObjStorage
+from swh.objstorage.backends.pathslicing import PathSlicingObjStorage
from swh.objstorage.multiplexer import MultiplexerObjStorage
from swh.objstorage.multiplexer.filter import add_filter, read_only
diff --git a/swh/objstorage/tests/test_objstorage_pathslicing.py b/swh/objstorage/tests/test_objstorage_pathslicing.py
--- a/swh/objstorage/tests/test_objstorage_pathslicing.py
+++ b/swh/objstorage/tests/test_objstorage_pathslicing.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2017 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -9,7 +9,9 @@
from unittest.mock import patch, DEFAULT
from swh.model import hashutil
-from swh.objstorage import exc, get_objstorage, ID_HASH_LENGTH
+from swh.objstorage import exc
+from swh.objstorage.factory import get_objstorage
+from swh.objstorage.objstorage import ID_HASH_LENGTH
from .objstorage_testing import ObjStorageTestFixture
diff --git a/swh/objstorage/tests/test_objstorage_random_generator.py b/swh/objstorage/tests/test_objstorage_random_generator.py
--- a/swh/objstorage/tests/test_objstorage_random_generator.py
+++ b/swh/objstorage/tests/test_objstorage_random_generator.py
@@ -1,10 +1,10 @@
-# Copyright (C) 2019 The Software Heritage developers
+# Copyright (C) 2019-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
from collections import Iterator
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
def test_random_generator_objstorage():
diff --git a/swh/objstorage/tests/test_objstorage_striping.py b/swh/objstorage/tests/test_objstorage_striping.py
--- a/swh/objstorage/tests/test_objstorage_striping.py
+++ b/swh/objstorage/tests/test_objstorage_striping.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2015-2016 The Software Heritage developers
+# Copyright (C) 2015-2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
@@ -8,7 +8,7 @@
import tempfile
import unittest
-from swh.objstorage import get_objstorage
+from swh.objstorage.factory import get_objstorage
from .objstorage_testing import ObjStorageTestFixture
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Jan 30 2025, 12:06 PM (6 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220094
Attached To
D3062: Move the content of swh/objstorage/__init__.py in swh/objstorage/factory.py
Event Timeline
Log In to Comment