diff --git a/swh/vault/api/server.py b/swh/vault/api/server.py index 91584c8..5b58d61 100644 --- a/swh/vault/api/server.py +++ b/swh/vault/api/server.py @@ -1,120 +1,121 @@ # Copyright (C) 2016-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import annotations import os from typing import Any, Dict, Optional from swh.core.api import RPCServerApp from swh.core.api import encode_data_server as encode_data from swh.core.api import error_handler from swh.core.config import config_basepath, merge_configs, read_raw_config from swh.vault import get_vault as get_swhvault from swh.vault.backend import NotFoundExc from swh.vault.interface import VaultInterface from .serializers import DECODERS, ENCODERS # do not define default services here DEFAULT_CONFIG = { "client_max_size": 1024**3, } def get_vault(): global vault if not vault: vault = get_swhvault(**app.config["vault"]) return vault class VaultServerApp(RPCServerApp): extra_type_decoders = DECODERS extra_type_encoders = ENCODERS vault = None app = VaultServerApp( __name__, backend_class=VaultInterface, backend_factory=get_vault, ) @app.errorhandler(NotFoundExc) def argument_error_handler(exception): return error_handler(exception, encode_data, status_code=400) @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.route("/") def index(): return "SWH Vault API server" def check_config(cfg: Dict[str, Any]) -> Dict[str, Any]: - """Ensure the configuration is ok to run a local vault server, and propagate defaults. + """Ensure the configuration is ok to run a postgresql vault server, and propagate + defaults. Raises: - EnvironmentError if the configuration is not for local instance + EnvironmentError if the configuration is not for postgresql instance ValueError if one of the following keys is missing: vault, cache, storage, scheduler Returns: - New configuration dict to instantiate a local vault server instance. + New configuration dict to instantiate a postgresql vault server instance. """ cfg = cfg.copy() if "vault" not in cfg: raise ValueError("missing 'vault' configuration") vcfg = cfg["vault"] - if vcfg["cls"] != "local": + if vcfg["cls"] not in ("local", "postgresql"): raise EnvironmentError( - "The vault backend can only be started with a 'local' configuration", + "The vault backend can only be started with a 'postgresql' configuration", ) # TODO: Soft-deprecation of args key. Remove when ready. vcfg.update(vcfg.get("args", {})) # Default to top-level value if any vcfg = {**cfg, **vcfg} for key in ("cache", "storage", "scheduler"): if not vcfg.get(key): raise ValueError(f"invalid configuration: missing {key} config entry.") return vcfg def make_app_from_configfile( config_path: Optional[str] = None, **kwargs ) -> VaultServerApp: """Load and check configuration if ok, then instantiate (once) a vault server application. """ config_path = os.environ.get("SWH_CONFIG_FILENAME", config_path) if not config_path: raise ValueError("Missing configuration path.") if not os.path.isfile(config_path): raise ValueError(f"Configuration path {config_path} should exist.") app_config = read_raw_config(config_basepath(config_path)) app_config["vault"] = check_config(app_config) app.config.update(merge_configs(DEFAULT_CONFIG, app_config)) return app if __name__ == "__main__": print("Deprecated. Use swh-vault ") diff --git a/swh/vault/tests/test_server.py b/swh/vault/tests/test_server.py index 184f426..1f67ccf 100644 --- a/swh/vault/tests/test_server.py +++ b/swh/vault/tests/test_server.py @@ -1,184 +1,187 @@ # Copyright (C) 2020-2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import copy import os from typing import Any, Dict import pytest import yaml from swh.core.api.serializers import json_dumps, msgpack_dumps, msgpack_loads from swh.vault.api.serializers import ENCODERS import swh.vault.api.server from swh.vault.api.server import app, check_config, get_vault, make_app_from_configfile from swh.vault.tests.test_backend import TEST_SWHID @pytest.fixture def swh_vault_server_config(swh_vault_config: Dict[str, Any]) -> Dict[str, Any]: """Returns a vault server configuration, with ``storage``, ``scheduler`` and ``cache`` set at the toplevel""" return { - "vault": {"cls": "local", "db": swh_vault_config["db"]}, + "vault": {"cls": "postgresql", "db": swh_vault_config["db"]}, "client_max_size": 1024**3, **{k: v for k, v in swh_vault_config.items() if k != "db"}, } @pytest.fixture def swh_vault_server_config_file(swh_vault_server_config, monkeypatch, tmp_path): """Creates a vault server configuration file and sets it into SWH_CONFIG_FILENAME""" conf_path = os.path.join(str(tmp_path), "vault-server.yml") with open(conf_path, "w") as f: f.write(yaml.dump(swh_vault_server_config)) monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) return conf_path def test_make_app_from_file_missing(): with pytest.raises(ValueError, match="Missing configuration path."): make_app_from_configfile() def test_make_app_from_file_does_not_exist(tmp_path): conf_path = os.path.join(str(tmp_path), "vault-server.yml") assert os.path.exists(conf_path) is False with pytest.raises( ValueError, match=f"Configuration path {conf_path} should exist." ): make_app_from_configfile(conf_path) def test_make_app_from_env_variable(swh_vault_server_config_file): """Server initialization happens through env variable when no path is provided""" app = make_app_from_configfile() assert app is not None assert get_vault() is not None # Cleanup app del app.config["vault"] swh.vault.api.server.vault = None def test_make_app_from_file(swh_vault_server_config, tmp_path): """Server initialization happens through path if provided""" conf_path = os.path.join(str(tmp_path), "vault-server.yml") with open(conf_path, "w") as f: f.write(yaml.dump(swh_vault_server_config)) app = make_app_from_configfile(conf_path) assert app is not None assert get_vault() is not None # Cleanup app del app.config["vault"] swh.vault.api.server.vault = None @pytest.fixture def vault_app(swh_vault_server_config_file): yield make_app_from_configfile() # Cleanup app del app.config["vault"] swh.vault.api.server.vault = None @pytest.fixture def cli(vault_app): cli = vault_app.test_client() return cli def test_client_index(cli): resp = cli.get("/") assert resp.status == "200 OK" def test_client_cook_notfound(cli): resp = cli.post( "/cook", data=json_dumps( {"bundle_type": "flat", "swhid": TEST_SWHID}, extra_encoders=ENCODERS ), headers=[("Content-Type", "application/json")], ) assert resp.status == "400 BAD REQUEST" content = msgpack_loads(resp.data) assert content["type"] == "NotFoundExc" assert content["args"] == [f"flat {TEST_SWHID} was not found."] def test_client_progress_notfound(cli): resp = cli.post( "/progress", data=json_dumps( {"bundle_type": "flat", "swhid": TEST_SWHID}, extra_encoders=ENCODERS ), headers=[("Content-Type", "application/json")], ) assert resp.status == "400 BAD REQUEST" content = msgpack_loads(resp.data) assert content["type"] == "NotFoundExc" assert content["args"] == [f"flat {TEST_SWHID} was not found."] def test_client_batch_cook_invalid_type(cli): resp = cli.post( "/batch_cook", data=msgpack_dumps({"batch": [("foobar", [])]}), headers={"Content-Type": "application/x-msgpack"}, ) assert resp.status == "400 BAD REQUEST" content = msgpack_loads(resp.data) assert content["type"] == "NotFoundExc" assert content["args"] == ["foobar is an unknown type."] def test_client_batch_progress_notfound(cli): resp = cli.post( "/batch_progress", data=msgpack_dumps({"batch_id": 1}), headers={"Content-Type": "application/x-msgpack"}, ) assert resp.status == "400 BAD REQUEST" content = msgpack_loads(resp.data) assert content["type"] == "NotFoundExc" assert content["args"] == ["Batch 1 does not exist."] def test_check_config_missing_vault_configuration() -> None: """Irrelevant configuration file path raises""" with pytest.raises(ValueError, match="missing 'vault' configuration"): check_config({}) def test_check_config_not_local() -> None: """Wrong configuration raises""" expected_error = ( - "The vault backend can only be started with a 'local' configuration" + "The vault backend can only be started with a 'postgresql' configuration" ) with pytest.raises(EnvironmentError, match=expected_error): check_config({"vault": {"cls": "remote"}}) -def test_check_config_ok(swh_vault_server_config) -> None: +@pytest.mark.parametrize("clazz", ["local", "postgresql"]) +def test_check_config_ok(swh_vault_server_config, clazz) -> None: """Check that the default config is accepted""" + config = swh_vault_server_config.copy() + config["vault"]["cls"] = clazz assert check_config(swh_vault_server_config) is not None @pytest.mark.parametrize("missing_key", ["storage", "cache", "scheduler"]) def test_check_config_missing_key(missing_key, swh_vault_server_config) -> None: """Check that configs with a missing key get rejected""" config_ok = swh_vault_server_config config_ko = copy.deepcopy(config_ok) config_ko["vault"].pop(missing_key, None) config_ko.pop(missing_key, None) expected_error = f"invalid configuration: missing {missing_key} config entry" with pytest.raises(ValueError, match=expected_error): check_config(config_ko)