diff --git a/setup.py b/setup.py --- a/setup.py +++ b/setup.py @@ -53,6 +53,10 @@ extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, + entry_points=''' + [console_scripts] + swh-objstorage=swh.objstorage.cli:main + ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py --- a/swh/objstorage/api/server.py +++ b/swh/objstorage/api/server.py @@ -1,13 +1,13 @@ -# Copyright (C) 2015-2017 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import asyncio import aiohttp.web -import click +import os -from swh.core import config +from swh.core.config import read as config_read from swh.core.api_async import (SWHRemoteAPI, decode_request, encode_data_server as encode_data) from swh.model import hashutil @@ -15,17 +15,6 @@ from swh.objstorage.exc import ObjNotFoundError -DEFAULT_CONFIG_PATH = 'objstorage/server' -DEFAULT_CONFIG = { - 'cls': ('str', 'pathslicing'), - 'args': ('dict', { - 'root': '/srv/softwareheritage/objects', - 'slicing': '0:2/2:4/4:6', - }), - 'client_max_size': ('int', 1024 * 1024 * 1024), -} - - @asyncio.coroutine def index(request): return aiohttp.web.Response(body="SWH Objstorage API server") @@ -129,53 +118,103 @@ return response -@asyncio.coroutine -def set_app_config(app): - if app['config']: - cfg = app['config'] - else: - cfg = config.load_named_config(DEFAULT_CONFIG_PATH, DEFAULT_CONFIG) - if 'client_max_size' in cfg: - app._client_max_size = cfg.pop('client_max_size') - app.update(cfg) +def make_app(config): + """Initialize the remote api application. - -@asyncio.coroutine -def create_objstorage(app): - app['objstorage'] = get_objstorage(app['cls'], app['args']) - - -app = SWHRemoteAPI() -app['config'] = None -app.router.add_route('GET', '/', index) -app.router.add_route('POST', '/check_config', check_config) -app.router.add_route('POST', '/content/contains', contains) -app.router.add_route('POST', '/content/add', add_bytes) -app.router.add_route('POST', '/content/add/batch', add_batch) -app.router.add_route('POST', '/content/get', get_bytes) -app.router.add_route('POST', '/content/get/batch', get_batch) -app.router.add_route('POST', '/content/get/random', get_random_contents) -app.router.add_route('POST', '/content/check', check) -app.router.add_route('POST', '/content/delete', delete) -app.router.add_route('POST', '/content/add_stream/{hex_id}', add_stream) -app.router.add_route('GET', '/content/get_stream/{hex_id}', get_stream) -app.on_startup.append(set_app_config) -app.on_startup.append(create_objstorage) - - -@click.command() -@click.argument('config-path', required=1) -@click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5003, type=click.INT, - help="Binding port of the server") -@click.option('--debug/--nodebug', default=True, - help="Indicates if the server should run in debug mode") -def launch(config_path, host, port, debug): - cfg = config.load_named_config(config_path, DEFAULT_CONFIG) - app['config'] = cfg - app.update(debug=bool(debug)) - aiohttp.web.run_app(app, host=host, port=int(port)) + """ + app = SWHRemoteAPI() + # retro compatibility configuration settings + app['config'] = config + if 'objstorage' in config: + _cfg = config['objstorage'] + else: + _cfg = config + app['objstorage'] = get_objstorage(_cfg['cls'], _cfg['args']) + + if 'client_max_size' in config: + app._client_max_size = config.pop('client_max_size') + app.update(config) + + app.router.add_route('GET', '/', index) + app.router.add_route('POST', '/check_config', check_config) + app.router.add_route('POST', '/content/contains', contains) + app.router.add_route('POST', '/content/add', add_bytes) + app.router.add_route('POST', '/content/add/batch', add_batch) + app.router.add_route('POST', '/content/get', get_bytes) + app.router.add_route('POST', '/content/get/batch', get_batch) + app.router.add_route('POST', '/content/get/random', get_random_contents) + app.router.add_route('POST', '/content/check', check) + app.router.add_route('POST', '/content/delete', delete) + app.router.add_route('POST', '/content/add_stream/{hex_id}', add_stream) + app.router.add_route('GET', '/content/get_stream/{hex_id}', get_stream) + return app + + +def load_and_check_config(config_file): + """Check the minimal configuration is set to run the api or raise an + error explanation. + + Args: + config_file (str): Path to the configuration file to load + type (str): configuration type. For 'local' type, more + checks are done. + + Raises: + Error if the setup is not as expected + + Returns: + configuration as a dict + + """ + if not config_file: + raise EnvironmentError('Configuration file must be defined') + + if not os.path.exists(config_file): + raise FileNotFoundError('Configuration file %s does not exist' % ( + config_file, )) + + cfg = config_read(config_file) + + if 'objstorage' not in cfg: + raise KeyError( + "Invalid configuration; missing objstorage config entry") + + missing_keys = [] + vcfg = cfg['objstorage'] + for key in ('cls', 'args'): + v = vcfg.get(key) + if v is None: + missing_keys.append(key) + + if missing_keys: + raise KeyError( + "Invalid configuration; missing %s config entry" % ( + ', '.join(missing_keys), )) + + cls = vcfg.get('cls') + if cls == 'pathslicing': + args = vcfg['args'] + for key in ('root', 'slicing'): + v = args.get(key) + if v is None: + missing_keys.append(key) + + if missing_keys: + raise KeyError( + "Invalid configuration; missing args.%s config entry" % ( + ', '.join(missing_keys), )) + + return cfg + + +def make_app_from_configfile(): + """Load configuration and then build application to run + + """ + config_file = os.environ.get('SWH_CONFIG_FILENAME') + config = load_and_check_config(config_file) + return make_app(config=config) if __name__ == '__main__': - launch() + print('Deprecated. Use swh-objstorage') diff --git a/swh/objstorage/api/wsgi.py b/swh/objstorage/api/wsgi.py new file mode 100644 --- /dev/null +++ b/swh/objstorage/api/wsgi.py @@ -0,0 +1,8 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from .server import make_app_from_configfile + +application = make_app_from_configfile() diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py new file mode 100644 --- /dev/null +++ b/swh/objstorage/cli.py @@ -0,0 +1,27 @@ +# Copyright (C) 2015-2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import click +import aiohttp.web + +from swh.objstorage.api.server import load_and_check_config, make_app + + +@click.command() +@click.argument('config-path', required=1) +@click.option('--host', default='0.0.0.0', help="Host to run the server") +@click.option('--port', default=5007, type=click.INT, + help="Binding port of the server") +@click.option('--debug/--nodebug', default=True, + help="Indicates if the server should run in debug mode") +def main(config_path, host, port, debug): + cfg = load_and_check_config(config_path) + app = make_app(cfg) + app.update(debug=bool(debug)) + aiohttp.web.run_app(app, host=host, port=int(port)) + + +if __name__ == '__main__': + main() diff --git a/swh/objstorage/tests/test_objstorage_api.py b/swh/objstorage/tests/test_objstorage_api.py --- a/swh/objstorage/tests/test_objstorage_api.py +++ b/swh/objstorage/tests/test_objstorage_api.py @@ -1,4 +1,4 @@ -# Copyright (C) 2015-2018 The Software Heritage developers +# Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -9,7 +9,7 @@ from swh.core.tests.server_testing import ServerTestFixtureAsync from swh.objstorage import get_objstorage -from swh.objstorage.api.server import app +from swh.objstorage.api.server import make_app from swh.objstorage.tests.objstorage_testing import ObjStorageTestFixture @@ -30,8 +30,7 @@ 'client_max_size': 8 * 1024 * 1024, } - self.app = app - self.app['config'] = self.config + self.app = make_app(self.config) super().setUp() self.storage = get_objstorage('remote', { 'url': self.url() diff --git a/swh/objstorage/tests/test_server.py b/swh/objstorage/tests/test_server.py new file mode 100644 --- /dev/null +++ b/swh/objstorage/tests/test_server.py @@ -0,0 +1,134 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import copy +import pytest +import yaml + +from swh.objstorage.api.server import load_and_check_config + + +def prepare_config_file(tmpdir, content, name='config.yml'): + """Prepare configuration file in `$tmpdir/name` with content `content`. + + Args: + tmpdir (LocalPath): root directory + content (str/dict): Content of the file either as string or as a dict. + If a dict, converts the dict into a yaml string. + name (str): configuration filename + + Returns + path (str) of the configuration file prepared. + + """ + config_path = tmpdir / name + if isinstance(content, dict): # convert if needed + content = yaml.dump(content) + config_path.write_text(content, encoding='utf-8') + # pytest on python3.5 does not support LocalPath manipulation, so + # convert path to string + return str(config_path) + + +def test_load_and_check_config_no_configuration(): + """Inexistant configuration files raises""" + with pytest.raises(EnvironmentError) as e: + load_and_check_config(None) + + assert e.value.args[0] == 'Configuration file must be defined' + + config_path = '/indexer/inexistant/config.yml' + with pytest.raises(FileNotFoundError) as e: + load_and_check_config(config_path) + + assert e.value.args[0] == 'Configuration file %s does not exist' % ( + config_path, ) + + +def test_load_and_check_config_invalid_configuration_toplevel(tmpdir): + """Invalid configuration raises""" + config = { + 'something': 'useless' + } + config_path = prepare_config_file(tmpdir, content=config) + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert ( + e.value.args[0] == + 'Invalid configuration; missing objstorage config entry' + ) + + +def test_load_and_check_config_invalid_configuration(tmpdir): + """Invalid configuration raises""" + for data, missing_keys in [ + ({'objstorage': {'something': 'useless'}}, ['cls', 'args']), + ({'objstorage': {'cls': 'something'}}, ['args']), + ]: + config_path = prepare_config_file(tmpdir, content=data) + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert ( + e.value.args[0] == + 'Invalid configuration; missing %s config entry' % ( + ', '.join(missing_keys), ) + ) + + +def test_load_and_check_config_invalid_configuration_level2(tmpdir): + """Invalid configuration at 2nd level raises""" + config = { + 'objstorage': { + 'cls': 'pathslicing', + 'args': { + 'root': 'root', + 'slicing': 'slicing', + }, + 'client_max_size': '10', + } + } + for key in ('root', 'slicing'): + c = copy.deepcopy(config) + c['objstorage']['args'].pop(key) + config_path = prepare_config_file(tmpdir, c) + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert ( + e.value.args[0] == + "Invalid configuration; missing args.%s config entry" % key + ) + + +def test_load_and_check_config_fine(tmpdir): + """pathslicing configuration fine loads ok""" + config = { + 'objstorage': { + 'cls': 'pathslicing', + 'args': { + 'root': 'root', + 'slicing': 'slicing', + } + } + } + + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path) + assert cfg == config + + +def test_load_and_check_config_fine2(tmpdir): + config = { + 'client_max_size': '10', + 'objstorage': { + 'cls': 'remote', + 'args': {} + } + } + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path) + assert cfg == config