Software Heritage storage server

diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py index 4a0ea0c2..cc6634ae 100644 --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -1,132 +1,118 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging from swh.core import config from swh.storage import get_storage as get_swhstorage from swh.core.api import (RPCServerApp, error_handler, encode_data_server as encode_data) from ..storage import Storage from ..metrics import timed def get_storage(): global storage if not storage: storage = get_swhstorage(**app.config['storage']) return storage app = RPCServerApp(__name__, backend_class=Storage, backend_factory=get_storage) storage = None @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.route('/') @timed def index(): return ''' Software Heritage storage server

You have reached the Software Heritage storage server.
See its documentation and API for more information

''' @app.route('/stat/counters', methods=['GET']) @timed def stat_counters(): return encode_data(get_storage().stat_counters()) @app.route('/stat/refresh', methods=['GET']) @timed def refresh_stat_counters(): return encode_data(get_storage().refresh_stat_counters()) api_cfg = None def load_and_check_config(config_file, type='local'): """Check the minimal configuration is set to run the api or raise an error explanation. Args: config_file (str): Path to the configuration file to load type (str): configuration type. For 'local' type, more checks are done. Raises: Error if the setup is not as expected Returns: configuration as a dict """ if not config_file: raise EnvironmentError('Configuration file must be defined') if not os.path.exists(config_file): raise FileNotFoundError('Configuration file %s does not exist' % ( config_file, )) cfg = config.read(config_file) if 'storage' not in cfg: raise KeyError("Missing '%storage' configuration") - if type == 'local': - vcfg = cfg['storage'] - cls = vcfg.get('cls') - if cls != 'local': - raise ValueError( - "The storage backend can only be started with a 'local' " - "configuration") - - args = vcfg['args'] - for key in ('db', 'objstorage'): - if not args.get(key): - raise ValueError( - "Invalid configuration; missing '%s' config entry" % key) - return cfg def make_app_from_configfile(): """Run the WSGI app from the webserver, loading the configuration from a configuration file. SWH_CONFIG_FILENAME environment variable defines the configuration path to load. """ global api_cfg if not api_cfg: config_file = os.environ.get('SWH_CONFIG_FILENAME') api_cfg = load_and_check_config(config_file) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) return app if __name__ == '__main__': print('Deprecated. Use swh-storage') diff --git a/swh/storage/tests/test_server.py b/swh/storage/tests/test_server.py index 28ab7df3..a412c5a1 100644 --- a/swh/storage/tests/test_server.py +++ b/swh/storage/tests/test_server.py @@ -1,129 +1,85 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import copy import pytest import yaml from swh.storage.api.server import load_and_check_config def prepare_config_file(tmpdir, content, name='config.yml'): """Prepare configuration file in `$tmpdir/name` with content `content`. Args: tmpdir (LocalPath): root directory content (str/dict): Content of the file either as string or as a dict. If a dict, converts the dict into a yaml string. name (str): configuration filename Returns path (str) of the configuration file prepared. """ config_path = tmpdir / name if isinstance(content, dict): # convert if needed content = yaml.dump(content) config_path.write_text(content, encoding='utf-8') # pytest on python3.5 does not support LocalPath manipulation, so # convert path to string return str(config_path) def test_load_and_check_config_no_configuration(): """Inexistent configuration files raises""" with pytest.raises(EnvironmentError) as e: load_and_check_config(None) assert e.value.args[0] == 'Configuration file must be defined' config_path = '/some/inexistent/config.yml' with pytest.raises(FileNotFoundError) as e: load_and_check_config(config_path) assert e.value.args[0] == 'Configuration file %s does not exist' % ( config_path, ) def test_load_and_check_config_wrong_configuration(tmpdir): """Wrong configuration raises""" config_path = prepare_config_file(tmpdir, 'something: useless') with pytest.raises(KeyError) as e: load_and_check_config(config_path) assert e.value.args[0] == 'Missing \'%storage\' configuration' -def test_load_and_check_config_remote_config_local_type_raise(tmpdir): - """'local' configuration without 'local' storage raises""" - config = { - 'storage': { - 'cls': 'remote', - 'args': {} - } - } - config_path = prepare_config_file(tmpdir, config) - with pytest.raises(ValueError) as e: - load_and_check_config(config_path, type='local') - - assert ( - e.value.args[0] == - "The storage backend can only be started with a 'local' configuration" - ) - - -def test_load_and_check_config_local_incomplete_configuration(tmpdir): - """Incomplete 'local' configuration should raise""" - config = { - 'storage': { - 'cls': 'local', - 'args': { - 'db': 'database', - 'objstorage': 'object_storage' - } - } - } - - for key in ('db', 'objstorage'): - c = copy.deepcopy(config) - c['storage']['args'].pop(key) - config_path = prepare_config_file(tmpdir, c) - with pytest.raises(ValueError) as e: - load_and_check_config(config_path) - - assert ( - e.value.args[0] == - "Invalid configuration; missing '%s' config entry" % key - ) - - def test_load_and_check_config_local_config_fine(tmpdir): """'Remote configuration is fine""" config = { 'storage': { 'cls': 'local', 'args': { 'db': 'db', 'objstorage': 'something', } } } config_path = prepare_config_file(tmpdir, config) cfg = load_and_check_config(config_path, type='local') assert cfg == config def test_load_and_check_config_remote_config_fine(tmpdir): """'Remote configuration is fine""" config = { 'storage': { 'cls': 'remote', 'args': {} } } config_path = prepare_config_file(tmpdir, config) cfg = load_and_check_config(config_path, type='any') assert cfg == config