Page MenuHomeSoftware Heritage
Files F7066649

D1170.diff
No OneTemporary
Actions

D1170.diff
View Options

diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py
--- a/swh/indexer/storage/api/server.py
+++ b/swh/indexer/storage/api/server.py
@@ -1,10 +1,10 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import os
import logging
-import click
from swh.core import config
from swh.core.api import (SWHServerAPIApp, error_handler,
@@ -14,17 +14,6 @@
)
-DEFAULT_CONFIG_PATH = 'storage/indexer'
-DEFAULT_CONFIG = {
- INDEXER_CFG_KEY: ('dict', {
- 'cls': 'local',
- 'args': {
- 'db': 'dbname=softwareheritage-indexer-dev',
- },
- })
-}
-
-
def get_storage():
global storage
if not storage:
@@ -52,17 +41,64 @@
api_cfg = None
-def run_from_webserver(environ, start_response,
- config_path=DEFAULT_CONFIG_PATH):
- """Run the WSGI app from the webserver, loading the configuration."""
+def load_and_check_config(config_file, type='local'):
+ """Check the minimal configuration is set to run the api or raise an
+ error explanation.
+
+ Args:
+ config_file (str): Path to the configuration file to load
+ type (str): configuration type. For 'local' type, more
+ checks are done.
+
+ Raises:
+ Error if the setup is not as expected
+
+ Returns:
+ configuration as a dict
+
+ """
+ if not config_file:
+ raise EnvironmentError('Configuration file must be defined')
+
+ if not os.path.exists(config_file):
+ raise FileNotFoundError('Configuration file %s does not exist' % (
+ config_file, ))
+
+ cfg = config.read(config_file)
+ if 'indexer_storage' not in cfg:
+ raise KeyError("Missing '%indexer_storage' configuration")
+
+ if type == 'local':
+ vcfg = cfg['indexer_storage']
+ cls = vcfg.get('cls')
+ if cls != 'local':
+ raise ValueError(
+ "The indexer_storage backend can only be started with a "
+ "'local' configuration")
+
+ args = vcfg['args']
+ if not args.get('db'):
+ raise ValueError(
+ "Invalid configuration; missing 'db' config entry")
+
+ return cfg
+
+
+def make_app_from_configfile():
+ """Run the WSGI app from the webserver, loading the configuration from
+ a configuration file.
+
+ SWH_CONFIG_FILENAME environment variable defines the
+ configuration path to load.
+
+ """
global api_cfg
if not api_cfg:
- api_cfg = config.load_named_config(config_path, DEFAULT_CONFIG)
+ config_file = os.environ.get('SWH_CONFIG_FILENAME')
+ api_cfg = load_and_check_config(config_file)
app.config.update(api_cfg)
handler = logging.StreamHandler()
app.logger.addHandler(handler)
- return app(environ, start_response)
-
@click.command()
@click.argument('config-path', required=1)
@@ -74,6 +110,7 @@
def launch(config_path, host, port, debug):
app.config.update(config.read(config_path, DEFAULT_CONFIG))
app.run(host, port=int(port), debug=bool(debug))
+ return app
if __name__ == '__main__':
diff --git a/swh/indexer/tests/storage/test_server.py b/swh/indexer/tests/storage/test_server.py
new file mode 100644
--- /dev/null
+++ b/swh/indexer/tests/storage/test_server.py
@@ -0,0 +1,123 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+import yaml
+
+from swh.indexer.storage.api.server import load_and_check_config
+
+
+def prepare_config_file(tmpdir, content, name='config.yml'):
+ """Prepare configuration file in `$tmpdir/name` with content `content`.
+
+ Args:
+ tmpdir (LocalPath): root directory
+ content (str/dict): Content of the file either as string or as a dict.
+ If a dict, converts the dict into a yaml string.
+ name (str): configuration filename
+
+ Returns
+ path (str) of the configuration file prepared.
+
+ """
+ config_path = tmpdir / name
+ if isinstance(content, dict): # convert if needed
+ content = yaml.dump(content)
+ config_path.write_text(content, encoding='utf-8')
+ # pytest on python3.5 does not support LocalPath manipulation, so
+ # convert path to string
+ return str(config_path)
+
+
+def test_load_and_check_config_no_configuration():
+ """Inexistant configuration files raises"""
+ with pytest.raises(EnvironmentError) as e:
+ load_and_check_config(None)
+
+ assert e.value.args[0] == 'Configuration file must be defined'
+
+ config_path = '/indexer/inexistant/config.yml'
+ with pytest.raises(FileNotFoundError) as e:
+ load_and_check_config(config_path)
+
+ assert e.value.args[0] == 'Configuration file %s does not exist' % (
+ config_path, )
+
+
+def test_load_and_check_config_wrong_configuration(tmpdir):
+ """Wrong configuration raises"""
+ config_path = prepare_config_file(tmpdir, 'something: useless')
+ with pytest.raises(KeyError) as e:
+ load_and_check_config(config_path)
+
+ assert e.value.args[0] == 'Missing \'%indexer_storage\' configuration'
+
+
+def test_load_and_check_config_remote_config_local_type_raise(tmpdir):
+ """'local' configuration without 'local' storage raises"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'remote',
+ 'args': {}
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ with pytest.raises(ValueError) as e:
+ load_and_check_config(config_path, type='local')
+
+ assert (
+ e.value.args[0] ==
+ "The indexer_storage backend can only be started with a 'local' "
+ "configuration"
+ )
+
+
+def test_load_and_check_config_local_incomplete_configuration(tmpdir):
+ """Incomplete 'local' configuration should raise"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'local',
+ 'args': {
+ }
+ }
+ }
+
+ config_path = prepare_config_file(tmpdir, config)
+ with pytest.raises(ValueError) as e:
+ load_and_check_config(config_path)
+
+ assert (
+ e.value.args[0] ==
+ "Invalid configuration; missing 'db' config entry"
+ )
+
+
+def test_load_and_check_config_local_config_fine(tmpdir):
+ """'Remote configuration is fine"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'local',
+ 'args': {
+ 'db': 'db',
+ }
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ cfg = load_and_check_config(config_path, type='local')
+ assert cfg == config
+
+
+def test_load_and_check_config_remote_config_fine(tmpdir):
+ """'Remote configuration is fine"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'remote',
+ 'args': {}
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ cfg = load_and_check_config(config_path, type='any')
+
+ assert cfg == config

File Metadata

Mime Type
text/plain
Expires
Nov 5 2024, 5:17 PM (12 w, 4 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3220287

Event Timeline

About · Developer information · Legal