diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -47,6 +47,10 @@ scripts=[ 'bin/swh-storage-add-dir', ], + entry_points=''' + [console_scripts] + swh-storage=swh.storage.cli:main + ''', setup_requires=['vcversioner'], install_requires=parse_requirements() + parse_requirements('swh'), extras_require={ diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -1,11 +1,11 @@ -# Copyright (C) 2015-2017 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os import json import logging -import click from flask import request @@ -15,24 +15,6 @@ error_handler, encode_data_server as encode_data) -DEFAULT_CONFIG_PATH = 'storage/storage' -DEFAULT_CONFIG = { - 'storage': ('dict', { - 'cls': 'local', - 'args': { - 'db': 'dbname=softwareheritage-dev', - 'objstorage': { - 'cls': 'pathslicing', - 'args': { - 'root': '/srv/softwareheritage/objects', - 'slicing': '0:2/2:4/4:6', - }, - }, - }, - }) -} - - app = SWHServerAPIApp(__name__) storage = None @@ -386,29 +368,67 @@ api_cfg = None -def run_from_webserver(environ, start_response, - config_path=DEFAULT_CONFIG_PATH): - """Run the WSGI app from the webserver, loading the configuration.""" +def load_and_check_config(config_file, type='local'): + """Check the minimal configuration is set to run the api or raise an + error explanation. + + Args: + config_file (str): Path to the configuration file to load + type (str): configuration type. For 'local' type, more + checks are done. + + Raises: + Error if the setup is not as expected + + Returns: + configuration as a dict + + """ + if not config_file: + raise EnvironmentError('Configuration file must be defined') + + if not os.path.exists(config_file): + raise FileNotFoundError('Configuration file %s does not exist' % ( + config_file, )) + + cfg = config.read(config_file) + if 'storage' not in cfg: + raise KeyError("Missing '%storage' configuration") + + if type == 'local': + vcfg = cfg['storage'] + cls = vcfg.get('cls') + if cls != 'local': + raise EnvironmentError( + "The storage backend can only be started with a 'local' " + "configuration") + + args = vcfg['args'] + for key in ('db', 'objstorage'): + if not args.get(key): + raise ValueError( + "Invalid configuration; missing '%s' config entry" % key) + + return cfg + + +def make_app_from_configfile(): + """Run the WSGI app from the webserver, loading the configuration from + a configuration file. + + SWH_CONFIG_FILENAME environment variable defines the + configuration path to load. + + """ global api_cfg if not api_cfg: - api_cfg = config.load_named_config(config_path, DEFAULT_CONFIG) + config_file = os.environ.get('SWH_CONFIG_FILENAME') + api_cfg = load_and_check_config(config_file) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) - return app(environ, start_response) - - -@click.command() -@click.argument('config-path', required=1) -@click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5002, type=click.INT, - help="Binding port of the server") -@click.option('--debug/--nodebug', default=True, - help="Indicates if the server should run in debug mode") -def launch(config_path, host, port, debug): - app.config.update(config.read(config_path, DEFAULT_CONFIG)) - app.run(host, port=int(port), debug=bool(debug)) + return app if __name__ == '__main__': - launch() + print('Deprecated. Use swh-storage') diff --git a/swh/storage/api/wsgi.py b/swh/storage/api/wsgi.py new file mode 100644 --- /dev/null +++ b/swh/storage/api/wsgi.py @@ -0,0 +1,8 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from .server import make_app_from_configfile + +application = make_app_from_configfile() diff --git a/swh/storage/cli.py b/swh/storage/cli.py new file mode 100644 --- /dev/null +++ b/swh/storage/cli.py @@ -0,0 +1,25 @@ +# Copyright (C) 2015-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import click + +from swh.storage.api.server import load_and_check_config, app + + +@click.command() +@click.argument('config-path', required=1) +@click.option('--host', default='0.0.0.0', help="Host to run the server") +@click.option('--port', default=5002, type=click.INT, + help="Binding port of the server") +@click.option('--debug/--nodebug', default=True, + help="Indicates if the server should run in debug mode") +def launch(config_path, host, port, debug): + api_cfg = load_and_check_config(config_path, type='any') + app.config.update(api_cfg) + app.run(host, port=int(port), debug=bool(debug)) + + +if __name__ == '__main__': + launch() diff --git a/swh/storage/tests/test_server.py b/swh/storage/tests/test_server.py new file mode 100644 --- /dev/null +++ b/swh/storage/tests/test_server.py @@ -0,0 +1,129 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import copy +import pytest +import yaml + +from swh.storage.api.server import load_and_check_config + + +def prepare_config_file(tmpdir, content, name='config.yml'): + """Prepare configuration file in `$tmpdir/name` with content `content`. + + Args: + tmpdir (LocalPath): root directory + content (str/dict): Content of the file either as string or as a dict. + If a dict, converts the dict into a yaml string. + name (str): configuration filename + + Returns + path (str) of the configuration file prepared. + + """ + config_path = tmpdir / name + if isinstance(content, dict): # convert if needed + content = yaml.dump(content) + config_path.write_text(content, encoding='utf-8') + # pytest on python3.5 does not support LocalPath manipulation, so + # convert path to string + return str(config_path) + + +def test_load_and_check_config_no_configuration(): + """Inexistant configuration files raises""" + with pytest.raises(EnvironmentError) as e: + load_and_check_config(None) + + assert e.value.args[0] == 'Configuration file must be defined' + + config_path = '/some/inexistant/config.yml' + with pytest.raises(FileNotFoundError) as e: + load_and_check_config(config_path) + + assert e.value.args[0] == 'Configuration file %s does not exist' % ( + config_path, ) + + +def test_load_and_check_config_wrong_configuration(tmpdir): + """Wrong configuration raises""" + config_path = prepare_config_file(tmpdir, 'something: useless') + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert e.value.args[0] == 'Missing \'%storage\' configuration' + + +def test_load_and_check_config_remote_config_local_type_raise(tmpdir): + """'local' configuration without 'local' storage raises""" + config = { + 'storage': { + 'cls': 'remote', + 'args': {} + } + } + config_path = prepare_config_file(tmpdir, config) + with pytest.raises(EnvironmentError) as e: + load_and_check_config(config_path, type='local') + + assert ( + e.value.args[0] == + "The storage backend can only be started with a 'local' configuration" + ) + + +def test_load_and_check_config_local_incomplete_configuration(tmpdir): + """Incomplete 'local' configuration should raise""" + config = { + 'storage': { + 'cls': 'local', + 'args': { + 'db': 'database', + 'objstorage': 'object_storage' + } + } + } + + for key in ('db', 'objstorage'): + c = copy.deepcopy(config) + c['storage']['args'].pop(key) + config_path = prepare_config_file(tmpdir, c) + with pytest.raises(ValueError) as e: + load_and_check_config(config_path) + + assert ( + e.value.args[0] == + "Invalid configuration; missing '%s' config entry" % key + ) + + +def test_load_and_check_config_local_config_fine(tmpdir): + """'Remote configuration is fine""" + config = { + 'storage': { + 'cls': 'local', + 'args': { + 'db': 'db', + 'objstorage': 'something', + } + } + } + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path, type='local') + assert cfg == config + + +def test_load_and_check_config_remote_config_fine(tmpdir): + """'Remote configuration is fine""" + config = { + 'storage': { + 'cls': 'remote', + 'args': {} + } + } + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path, type='any') + + assert cfg == config