diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py index 20d81e2..02bd645 100644 --- a/swh/objstorage/api/server.py +++ b/swh/objstorage/api/server.py @@ -1,189 +1,186 @@ # Copyright (C) 2015-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import contextlib import functools import logging import os from flask import request from swh.core.api import RPCServerApp from swh.core.api import encode_data_server as encode_data from swh.core.api import error_handler from swh.core.config import read as config_read from swh.core.statsd import statsd from swh.objstorage.exc import Error, ObjNotFoundError from swh.objstorage.factory import get_objstorage as get_swhobjstorage from swh.objstorage.interface import ObjStorageInterface from swh.objstorage.objstorage import DEFAULT_LIMIT def timed(f): @functools.wraps(f) def w(*a, **kw): with statsd.timed( "swh_objstorage_request_duration_seconds", tags={"endpoint": f.__name__} ): return f(*a, **kw) return w @contextlib.contextmanager def timed_context(f_name): with statsd.timed( "swh_objstorage_request_duration_seconds", tags={"endpoint": f_name} ): yield def get_objstorage(): global objstorage if objstorage is None: objstorage = get_swhobjstorage(**app.config["objstorage"]) return objstorage class ObjStorageServerApp(RPCServerApp): client_exception_classes = (ObjNotFoundError, Error) method_decorators = [timed] def pre_add(self, kw): """Called before the 'add' method.""" statsd.increment( "swh_objstorage_in_bytes_total", len(kw["content"]), tags={"endpoint": "add_bytes"}, ) def post_get(self, ret, kw): """Called after the 'get' method.""" statsd.increment( "swh_objstorage_out_bytes_total", len(ret), tags={"endpoint": "get_bytes"} ) app = ObjStorageServerApp( __name__, backend_class=ObjStorageInterface, backend_factory=get_objstorage, ) objstorage = None @app.errorhandler(Error) def argument_error_handler(exception): return error_handler(exception, encode_data, status_code=400) @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.route("/") @timed def index(): return "SWH Objstorage API server" @app.route("/content") def list_content(): last_obj_id = request.args.get("last_obj_id") if last_obj_id: last_obj_id = bytes.fromhex(last_obj_id) limit = int(request.args.get("limit", DEFAULT_LIMIT)) def generate(): with timed_context("list_content"): yield from get_objstorage().list_content(last_obj_id, limit=limit) return app.response_class(generate()) api_cfg = None def load_and_check_config(config_file): """Check the minimal configuration is set to run the api or raise an error explanation. Args: config_file (str): Path to the configuration file to load Raises: Error if the setup is not as expected Returns: configuration as a dict """ if not config_file: raise EnvironmentError("Configuration file must be defined") if not os.path.exists(config_file): raise FileNotFoundError("Configuration file %s does not exist" % (config_file,)) cfg = config_read(config_file) return validate_config(cfg) def validate_config(cfg): """Check the minimal configuration is set to run the api or raise an explanatory error. Args: cfg (dict): Loaded configuration. Raises: Error if the setup is not as expected Returns: configuration as a dict """ if "objstorage" not in cfg: raise KeyError("Invalid configuration; missing objstorage config entry") missing_keys = [] vcfg = cfg["objstorage"] if "cls" not in vcfg: raise KeyError("Invalid configuration; missing cls config entry") cls = vcfg["cls"] if cls == "pathslicing": - # Backwards-compatibility: either get the deprecated `args` from the - # objstorage config, or use the full config itself to check for keys - args = vcfg.get("args", vcfg) for key in ("root", "slicing"): - v = args.get(key) + v = vcfg.get(key) if v is None: missing_keys.append(key) if missing_keys: raise KeyError( "Invalid configuration; missing %s config entry" % (", ".join(missing_keys),) ) return cfg def make_app_from_configfile(): """Load configuration and then build application to run""" global api_cfg if not api_cfg: config_path = os.environ.get("SWH_CONFIG_FILENAME") api_cfg = load_and_check_config(config_path) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) return app if __name__ == "__main__": print("Deprecated. Use swh-objstorage") diff --git a/swh/objstorage/factory.py b/swh/objstorage/factory.py index aa21e8d..3ea563d 100644 --- a/swh/objstorage/factory.py +++ b/swh/objstorage/factory.py @@ -1,98 +1,87 @@ # Copyright (C) 2016-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import importlib -import warnings from swh.objstorage.interface import ObjStorageInterface from swh.objstorage.multiplexer import MultiplexerObjStorage, StripingObjStorage from swh.objstorage.multiplexer.filter import add_filters from swh.objstorage.objstorage import ObjStorage __all__ = ["get_objstorage", "ObjStorage"] OBJSTORAGE_IMPLEMENTATIONS = { "pathslicing": ".backends.pathslicing.PathSlicingObjStorage", "remote": ".api.client.RemoteObjStorage", "memory": ".backends.in_memory.InMemoryObjStorage", "seaweedfs": ".backends.seaweedfs.SeaweedFilerObjStorage", "random": ".backends.generator.RandomGeneratorObjStorage", "http": ".backends.http.HTTPReadOnlyObjStorage", "noop": ".backends.noop.NoopObjStorage", "azure": ".backends.azure.AzureCloudObjStorage", "azure-prefixed": ".backends.azure.PrefixedAzureCloudObjStorage", "s3": ".backends.libcloud.AwsCloudObjStorage", "swift": ".backends.libcloud.OpenStackCloudObjStorage", "winery": ".backends.winery.WineryObjStorage", } -def get_objstorage(cls: str, args=None, **kwargs) -> ObjStorageInterface: +def get_objstorage(cls: str, **kwargs) -> ObjStorageInterface: """Create an ObjStorage using the given implementation class. Args: cls: objstorage class unique key contained in the _STORAGE_CLASSES dict. kwargs: arguments for the required class of objstorage that must match exactly the one in the `__init__` method of the class. Returns: subclass of ObjStorage that match the given `storage_class` argument. Raises: ValueError: if the given storage class is not a valid objstorage key. """ - if args is not None: - warnings.warn( - 'Explicit "args" key is deprecated for objstorage initialization, ' - "use class arguments keys directly instead.", - DeprecationWarning, - ) - # TODO: when removing this, drop the "args" backwards compatibility - # from swh.objstorage.api.server configuration checker - kwargs = args - class_path = OBJSTORAGE_IMPLEMENTATIONS.get(cls) if class_path is None: raise ValueError( "Unknown storage class `%s`. Supported: %s" % (cls, ", ".join(OBJSTORAGE_IMPLEMENTATIONS)) ) if "." in class_path: (module_path, class_name) = class_path.rsplit(".", 1) try: module = importlib.import_module(module_path, package=__package__) except ImportError as e: raise ValueError(f"Storage class {cls} is not available: {e.args[0]}") ObjStorage = getattr(module, class_name) else: ObjStorage = globals()[class_path] return ObjStorage(**kwargs) def _construct_filtered_objstorage(storage_conf, filters_conf): return add_filters(get_objstorage(**storage_conf), filters_conf) OBJSTORAGE_IMPLEMENTATIONS["filtered"] = "_construct_filtered_objstorage" def _construct_multiplexer_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return MultiplexerObjStorage(storages) OBJSTORAGE_IMPLEMENTATIONS["multiplexer"] = "_construct_multiplexer_objstorage" def _construct_striping_objstorage(objstorages): storages = [get_objstorage(**conf) for conf in objstorages] return StripingObjStorage(storages) OBJSTORAGE_IMPLEMENTATIONS["striping"] = "_construct_striping_objstorage"