diff --git a/swh/scheduler/api/server.py b/swh/scheduler/api/server.py index 5ab7729..3cdb28d 100644 --- a/swh/scheduler/api/server.py +++ b/swh/scheduler/api/server.py @@ -1,204 +1,259 @@ -# Copyright (C) 2018 The Software Heritage developers +# Copyright (C) 2018-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import os import logging from flask import request, Flask from swh.core import config from swh.core.api import (decode_request, error_handler, encode_data_server as encode_data) from swh.core.api import negotiate, JSONFormatter, MsgpackFormatter from swh.scheduler import get_scheduler as get_scheduler_from -from swh.scheduler import DEFAULT_CONFIG, DEFAULT_CONFIG_PATH app = Flask(__name__) scheduler = None @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) def get_sched(): global scheduler if not scheduler: scheduler = get_scheduler_from(**app.config['scheduler']) return scheduler def has_no_empty_params(rule): return len(rule.defaults or ()) >= len(rule.arguments or ()) @app.route('/') @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def index(): return 'SWH Scheduler API server' @app.route('/close_connection', methods=['GET', 'POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def close_connection(): return get_sched().close_connection() @app.route('/set_status_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def set_status_tasks(): return get_sched().set_status_tasks(**decode_request(request)) @app.route('/create_task_type', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def create_task_type(): return get_sched().create_task_type(**decode_request(request)) @app.route('/get_task_type', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def get_task_type(): return get_sched().get_task_type(**decode_request(request)) @app.route('/get_task_types', methods=['GET', 'POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def get_task_types(): return get_sched().get_task_types(**decode_request(request)) @app.route('/create_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def create_tasks(): return get_sched().create_tasks(**decode_request(request)) @app.route('/disable_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def disable_tasks(): return get_sched().disable_tasks(**decode_request(request)) @app.route('/get_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def get_tasks(): return get_sched().get_tasks(**decode_request(request)) @app.route('/get_task_runs', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def get_task_runs(): return get_sched().get_task_runs(**decode_request(request)) @app.route('/search_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def search_tasks(): return get_sched().search_tasks(**decode_request(request)) @app.route('/peek_ready_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def peek_ready_tasks(): return get_sched().peek_ready_tasks(**decode_request(request)) @app.route('/grab_ready_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def grab_ready_tasks(): return get_sched().grab_ready_tasks(**decode_request(request)) @app.route('/schedule_task_run', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def schedule_task_run(): return get_sched().schedule_task_run(**decode_request(request)) @app.route('/mass_schedule_task_runs', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def mass_schedule_task_runs(): return get_sched().mass_schedule_task_runs(**decode_request(request)) @app.route('/start_task_run', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def start_task_run(): return get_sched().start_task_run(**decode_request(request)) @app.route('/end_task_run', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def end_task_run(): return get_sched().end_task_run(**decode_request(request)) @app.route('/filter_task_to_archive', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def filter_task_to_archive(): return get_sched().filter_task_to_archive(**decode_request(request)) @app.route('/delete_archived_tasks', methods=['POST']) @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def delete_archived_tasks(): return get_sched().delete_archived_tasks(**decode_request(request)) @app.route("/site-map") @negotiate(MsgpackFormatter) @negotiate(JSONFormatter) def site_map(): links = [] sched = get_sched() for rule in app.url_map.iter_rules(): if has_no_empty_params(rule) and hasattr(sched, rule.endpoint): links.append(dict( rule=rule.rule, description=getattr(sched, rule.endpoint).__doc__)) # links is now a list of url, endpoint tuples return links +def load_and_check_config(config_file, type='local'): + """Check the minimal configuration is set to run the api or raise an + error explanation. + + Args: + config_file (str): Path to the configuration file to load + type (str): configuration type. For 'local' type, more + checks are done. + + Raises: + Error if the setup is not as expected + + Returns: + configuration as a dict + + """ + if not config_file: + raise EnvironmentError('Configuration file must be defined') + + if not os.path.exists(config_file): + raise FileNotFoundError('Configuration file %s does not exist' % ( + config_file, )) + + cfg = config.read(config_file) + + vcfg = cfg.get('scheduler') + if not vcfg: + raise KeyError("Missing '%scheduler' configuration") + + if type == 'local': + cls = vcfg.get('cls') + if cls != 'local': + raise ValueError( + "The scheduler backend can only be started with a 'local' " + "configuration") + + args = vcfg.get('args') + if not args: + raise KeyError( + "Invalid configuration; missing 'args' config entry") + + db = args.get('db') + if not db: + raise KeyError( + "Invalid configuration; missing 'db' config entry") + + return cfg + + api_cfg = None -def run_from_webserver(environ, start_response, - config_path=DEFAULT_CONFIG_PATH): - """Run the WSGI app from the webserver, loading the configuration.""" +def make_app_from_configfile(): + """Run the WSGI app from the webserver, loading the configuration from + a configuration file. + + SWH_CONFIG_FILENAME environment variable defines the + configuration path to load. + + """ global api_cfg if not api_cfg: - api_cfg = config.load_named_config(config_path, DEFAULT_CONFIG) + config_file = os.environ.get('SWH_CONFIG_FILENAME') + api_cfg = load_and_check_config(config_file) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) - return app(environ, start_response) + return app if __name__ == '__main__': print('Please use the "swh-scheduler api-server" command') diff --git a/swh/scheduler/api/wsgi.py b/swh/scheduler/api/wsgi.py new file mode 100644 index 0000000..02c4901 --- /dev/null +++ b/swh/scheduler/api/wsgi.py @@ -0,0 +1,8 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from .server import make_app_from_configfile + +application = make_app_from_configfile() diff --git a/swh/scheduler/tests/test_server.py b/swh/scheduler/tests/test_server.py new file mode 100644 index 0000000..4d3ea5f --- /dev/null +++ b/swh/scheduler/tests/test_server.py @@ -0,0 +1,133 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import copy +import pytest +import yaml + +from swh.scheduler.api.server import load_and_check_config + + +def prepare_config_file(tmpdir, content, name='config.yml'): + """Prepare configuration file in `$tmpdir/name` with content `content`. + + Args: + tmpdir (LocalPath): root directory + content (str/dict): Content of the file either as string or as a dict. + If a dict, converts the dict into a yaml string. + name (str): configuration filename + + Returns + path (str) of the configuration file prepared. + + """ + config_path = tmpdir / name + if isinstance(content, dict): # convert if needed + content = yaml.dump(content) + config_path.write_text(content, encoding='utf-8') + # pytest on python3.5 does not support LocalPath manipulation, so + # convert path to string + return str(config_path) + + +def test_load_and_check_config_no_configuration(): + """Inexistant configuration files raises""" + with pytest.raises(EnvironmentError) as e: + load_and_check_config(None) + + assert e.value.args[0] == 'Configuration file must be defined' + + config_path = '/some/inexistant/config.yml' + with pytest.raises(FileNotFoundError) as e: + load_and_check_config(config_path) + + assert e.value.args[0] == 'Configuration file %s does not exist' % ( + config_path, ) + + +def test_load_and_check_config_wrong_configuration(tmpdir): + """Wrong configuration raises""" + config_path = prepare_config_file(tmpdir, 'something: useless') + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert e.value.args[0] == 'Missing \'%scheduler\' configuration' + + +def test_load_and_check_config_remote_config_local_type_raise(tmpdir): + """'local' configuration without 'local' storage raises""" + config = { + 'scheduler': { + 'cls': 'remote', + 'args': {} + } + } + config_path = prepare_config_file(tmpdir, config) + with pytest.raises(ValueError) as e: + load_and_check_config(config_path, type='local') + + assert ( + e.value.args[0] == + "The scheduler backend can only be started with a 'local'" + " configuration" + ) + + +def test_load_and_check_config_local_incomplete_configuration(tmpdir): + """Incomplete 'local' configuration should raise""" + config = { + 'scheduler': { + 'cls': 'local', + 'args': { + 'db': 'database', + 'something': 'needed-for-test', + } + } + } + + for key in ['db', 'args']: + c = copy.deepcopy(config) + if key == 'db': + source = c['scheduler']['args'] + else: + source = c['scheduler'] + source.pop(key) + config_path = prepare_config_file(tmpdir, c) + with pytest.raises(KeyError) as e: + load_and_check_config(config_path) + + assert ( + e.value.args[0] == + "Invalid configuration; missing '%s' config entry" % key + ) + + +def test_load_and_check_config_local_config_fine(tmpdir): + """Local configuration is fine""" + config = { + 'scheduler': { + 'cls': 'local', + 'args': { + 'db': 'db', + } + } + } + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path, type='local') + assert cfg == config + + +def test_load_and_check_config_remote_config_fine(tmpdir): + """'Remote configuration is fine""" + config = { + 'scheduler': { + 'cls': 'remote', + 'args': {} + } + } + config_path = prepare_config_file(tmpdir, config) + cfg = load_and_check_config(config_path, type='any') + + assert cfg == config