diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py index 10d3784..a6d6f32 100644 --- a/swh/indexer/storage/api/server.py +++ b/swh/indexer/storage/api/server.py @@ -1,121 +1,120 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging import os from typing import Any, Dict, Optional from swh.core import config from swh.core.api import RPCServerApp from swh.core.api import encode_data_server as encode_data from swh.core.api import error_handler from swh.indexer.storage import INDEXER_CFG_KEY, get_indexer_storage from swh.indexer.storage.exc import IndexerStorageArgumentException from swh.indexer.storage.interface import IndexerStorageInterface from .serializers import DECODERS, ENCODERS def get_storage(): global storage if not storage: storage = get_indexer_storage(**app.config[INDEXER_CFG_KEY]) return storage class IndexerStorageServerApp(RPCServerApp): extra_type_decoders = DECODERS extra_type_encoders = ENCODERS app = IndexerStorageServerApp( __name__, backend_class=IndexerStorageInterface, backend_factory=get_storage ) storage = None @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.errorhandler(IndexerStorageArgumentException) def argument_error_handler(exception): return error_handler(exception, encode_data, status_code=400) @app.route("/") def index(): return "SWH Indexer Storage API server" api_cfg = None def load_and_check_config( config_path: Optional[str], type: str = "local" ) -> Dict[str, Any]: """Check the minimal configuration is set to run the api or raise an error explanation. Args: config_path: Path to the configuration file to load type: configuration type. For 'local' type, more checks are done. Raises: Error if the setup is not as expected Returns: configuration as a dict """ if not config_path: raise EnvironmentError("Configuration file must be defined") if not os.path.exists(config_path): raise FileNotFoundError(f"Configuration file {config_path} does not exist") cfg = config.read(config_path) if "indexer_storage" not in cfg: raise KeyError("Missing '%indexer_storage' configuration") if type == "local": vcfg = cfg["indexer_storage"] cls = vcfg.get("cls") if cls != "local": raise ValueError( "The indexer_storage backend can only be started with a " "'local' configuration" ) - args = vcfg["args"] - if not args.get("db"): + if not vcfg.get("db"): raise ValueError("Invalid configuration; missing 'db' config entry") return cfg def make_app_from_configfile(): """Run the WSGI app from the webserver, loading the configuration from a configuration file. SWH_CONFIG_FILENAME environment variable defines the configuration path to load. """ global api_cfg if not api_cfg: config_path = os.environ.get("SWH_CONFIG_FILENAME") api_cfg = load_and_check_config(config_path) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) return app if __name__ == "__main__": print("Deprecated. Use swh-indexer") diff --git a/swh/indexer/tests/storage/test_server.py b/swh/indexer/tests/storage/test_server.py index ed0040e..91cf037 100644 --- a/swh/indexer/tests/storage/test_server.py +++ b/swh/indexer/tests/storage/test_server.py @@ -1,92 +1,99 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import pytest import yaml from swh.indexer.storage.api.server import load_and_check_config def prepare_config_file(tmpdir, content, name="config.yml") -> str: """Prepare configuration file in `$tmpdir/name` with content `content`. Args: tmpdir (LocalPath): root directory content (str/dict): Content of the file either as string or as a dict. If a dict, converts the dict into a yaml string. name (str): configuration filename Returns path (str) of the configuration file prepared. """ config_path = tmpdir / name if isinstance(content, dict): # convert if needed content = yaml.dump(content) config_path.write_text(content, encoding="utf-8") # pytest on python3.5 does not support LocalPath manipulation, so # convert path to string return str(config_path) @pytest.mark.parametrize("config_path", [None, ""]) def test_load_and_check_config_no_configuration(config_path) -> None: - """Inexistent configuration files raises""" + """Irrelevant configuration file path raises""" with pytest.raises(EnvironmentError, match="Configuration file must be defined"): load_and_check_config(config_path) def test_load_and_check_inexistent_config_path() -> None: + """Inexistent configuration file raises""" config_path = "/indexer/inexistent/config.yml" expected_error = f"Configuration file {config_path} does not exist" with pytest.raises(FileNotFoundError, match=expected_error): load_and_check_config(config_path) def test_load_and_check_config_wrong_configuration(tmpdir) -> None: """Wrong configuration raises""" config_path = prepare_config_file(tmpdir, "something: useless") with pytest.raises(KeyError, match="Missing '%indexer_storage' configuration"): load_and_check_config(config_path) -def test_load_and_check_config_remote_config_local_type_raise(tmpdir) -> None: - """'local' configuration without 'local' storage raises""" - config = {"indexer_storage": {"cls": "remote", "args": {}}} - config_path = prepare_config_file(tmpdir, config) +@pytest.mark.parametrize("class_storage", ["remote", "memory"]) +def test_load_and_check_config_remote_config_local_type_raise( + class_storage, tmpdir +) -> None: + """Any other configuration than 'local' (the default) is rejected""" + assert class_storage != "local" + incompatible_config = {"indexer_storage": {"cls": class_storage}} + config_path = prepare_config_file(tmpdir, incompatible_config) expected_error = ( "The indexer_storage backend can only be started with a 'local' " "configuration" ) + with pytest.raises(ValueError, match=expected_error): + load_and_check_config(config_path) with pytest.raises(ValueError, match=expected_error): load_and_check_config(config_path, type="local") +def test_load_and_check_config_remote_config_fine(tmpdir) -> None: + """'Remote configuration is fine (when changing the default type)""" + config = {"indexer_storage": {"cls": "remote"}} + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path, type="any") + + assert cfg == config + + def test_load_and_check_config_local_incomplete_configuration(tmpdir) -> None: """Incomplete 'local' configuration should raise""" - config = {"indexer_storage": {"cls": "local", "args": {}}} + config = {"indexer_storage": {"cls": "local"}} expected_error = "Invalid configuration; missing 'db' config entry" config_path = prepare_config_file(tmpdir, config) with pytest.raises(ValueError, match=expected_error): load_and_check_config(config_path) def test_load_and_check_config_local_config_fine(tmpdir) -> None: - """'Remote configuration is fine""" - config = {"indexer_storage": {"cls": "local", "args": {"db": "db",}}} + """'Complete 'local' configuration is fine""" + config = {"indexer_storage": {"cls": "local", "db": "db",}} config_path = prepare_config_file(tmpdir, config) cfg = load_and_check_config(config_path, type="local") assert cfg == config - - -def test_load_and_check_config_remote_config_fine(tmpdir) -> None: - """'Remote configuration is fine""" - config = {"indexer_storage": {"cls": "remote", "args": {}}} - config_path = prepare_config_file(tmpdir, config) - cfg = load_and_check_config(config_path, type="any") - - assert cfg == config