Page MenuHomeSoftware Heritage
Files F7066327

D1170.id3695.diff
No OneTemporary
Actions

D1170.id3695.diff
View Options

diff --git a/conftest.py b/conftest.py
--- a/conftest.py
+++ b/conftest.py
@@ -4,3 +4,8 @@
# https://hypothesis.readthedocs.io/en/latest/settings.html#settings-profiles
settings.register_profile("fast", max_examples=5, deadline=5000)
settings.register_profile("slow", max_examples=20, deadline=5000)
+
+# Ignore the following modules because wsgi module fails as no
+# configuration file is found (--doctest-modules forces the module
+# loading)
+collect_ignore = ['swh/indexer/storage/api/wsgi.py']
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -50,6 +50,10 @@
extras_require={'testing': parse_requirements('test')},
vcversioner={},
include_package_data=True,
+ entry_points='''
+ [console_scripts]
+ swh-indexer=swh.indexer.cli:main
+ ''',
classifiers=[
"Programming Language :: Python :: 3",
"Intended Audience :: Developers",
diff --git a/swh/indexer/cli.py b/swh/indexer/cli.py
new file mode 100644
--- /dev/null
+++ b/swh/indexer/cli.py
@@ -0,0 +1,25 @@
+# Copyright (C) 2015-2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import click
+
+from swh.indexer.storage.api.server import load_and_check_config, app
+
+
+@click.command()
+@click.argument('config-path', required=1)
+@click.option('--host', default='0.0.0.0', help="Host to run the server")
+@click.option('--port', default=5007, type=click.INT,
+ help="Binding port of the server")
+@click.option('--debug/--nodebug', default=True,
+ help="Indicates if the server should run in debug mode")
+def main(config_path, host, port, debug):
+ api_cfg = load_and_check_config(config_path, type='any')
+ app.config.update(api_cfg)
+ app.run(host, port=int(port), debug=bool(debug))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/swh/indexer/storage/api/server.py b/swh/indexer/storage/api/server.py
--- a/swh/indexer/storage/api/server.py
+++ b/swh/indexer/storage/api/server.py
@@ -1,10 +1,10 @@
-# Copyright (C) 2015-2018 The Software Heritage developers
+# Copyright (C) 2015-2019 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
+import os
import logging
-import click
from swh.core import config
from swh.core.api import (SWHServerAPIApp, error_handler,
@@ -14,17 +14,6 @@
)
-DEFAULT_CONFIG_PATH = 'storage/indexer'
-DEFAULT_CONFIG = {
- INDEXER_CFG_KEY: ('dict', {
- 'cls': 'local',
- 'args': {
- 'db': 'dbname=softwareheritage-indexer-dev',
- },
- })
-}
-
-
def get_storage():
global storage
if not storage:
@@ -52,29 +41,66 @@
api_cfg = None
-def run_from_webserver(environ, start_response,
- config_path=DEFAULT_CONFIG_PATH):
- """Run the WSGI app from the webserver, loading the configuration."""
+def load_and_check_config(config_file, type='local'):
+ """Check the minimal configuration is set to run the api or raise an
+ error explanation.
+
+ Args:
+ config_file (str): Path to the configuration file to load
+ type (str): configuration type. For 'local' type, more
+ checks are done.
+
+ Raises:
+ Error if the setup is not as expected
+
+ Returns:
+ configuration as a dict
+
+ """
+ if not config_file:
+ raise EnvironmentError('Configuration file must be defined')
+
+ if not os.path.exists(config_file):
+ raise FileNotFoundError('Configuration file %s does not exist' % (
+ config_file, ))
+
+ cfg = config.read(config_file)
+ if 'indexer_storage' not in cfg:
+ raise KeyError("Missing '%indexer_storage' configuration")
+
+ if type == 'local':
+ vcfg = cfg['indexer_storage']
+ cls = vcfg.get('cls')
+ if cls != 'local':
+ raise EnvironmentError(
+ "The indexer_storage backend can only be started with a "
+ "'local' configuration")
+
+ args = vcfg['args']
+ if not args.get('db'):
+ raise ValueError(
+ "Invalid configuration; missing 'db' config entry")
+
+ return cfg
+
+
+def make_app_from_configfile():
+ """Run the WSGI app from the webserver, loading the configuration from
+ a configuration file.
+
+ SWH_CONFIG_FILENAME environment variable defines the
+ configuration path to load.
+
+ """
global api_cfg
if not api_cfg:
- api_cfg = config.load_named_config(config_path, DEFAULT_CONFIG)
+ config_file = os.environ.get('SWH_CONFIG_FILENAME')
+ api_cfg = load_and_check_config(config_file)
app.config.update(api_cfg)
handler = logging.StreamHandler()
app.logger.addHandler(handler)
- return app(environ, start_response)
-
-
-@click.command()
-@click.argument('config-path', required=1)
-@click.option('--host', default='0.0.0.0', help="Host to run the server")
-@click.option('--port', default=5007, type=click.INT,
- help="Binding port of the server")
-@click.option('--debug/--nodebug', default=True,
- help="Indicates if the server should run in debug mode")
-def launch(config_path, host, port, debug):
- app.config.update(config.read(config_path, DEFAULT_CONFIG))
- app.run(host, port=int(port), debug=bool(debug))
+ return app
if __name__ == '__main__':
- launch()
+ print('Deprecated. Use swh-indexer')
diff --git a/swh/indexer/storage/api/wsgi.py b/swh/indexer/storage/api/wsgi.py
new file mode 100644
--- /dev/null
+++ b/swh/indexer/storage/api/wsgi.py
@@ -0,0 +1,8 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+from .server import make_app_from_configfile
+
+application = make_app_from_configfile()
diff --git a/swh/indexer/tests/storage/test_server.py b/swh/indexer/tests/storage/test_server.py
new file mode 100644
--- /dev/null
+++ b/swh/indexer/tests/storage/test_server.py
@@ -0,0 +1,123 @@
+# Copyright (C) 2019 The Software Heritage developers
+# See the AUTHORS file at the top-level directory of this distribution
+# License: GNU General Public License version 3, or any later version
+# See top-level LICENSE file for more information
+
+import pytest
+import yaml
+
+from swh.indexer.storage.api.server import load_and_check_config
+
+
+def prepare_config_file(tmpdir, content, name='config.yml'):
+ """Prepare configuration file in `$tmpdir/name` with content `content`.
+
+ Args:
+ tmpdir (LocalPath): root directory
+ content (str/dict): Content of the file either as string or as a dict.
+ If a dict, converts the dict into a yaml string.
+ name (str): configuration filename
+
+ Returns
+ path (str) of the configuration file prepared.
+
+ """
+ config_path = tmpdir / name
+ if isinstance(content, dict): # convert if needed
+ content = yaml.dump(content)
+ config_path.write_text(content, encoding='utf-8')
+ # pytest on python3.5 does not support LocalPath manipulation, so
+ # convert path to string
+ return str(config_path)
+
+
+def test_load_and_check_config_no_configuration():
+ """Inexistant configuration files raises"""
+ with pytest.raises(EnvironmentError) as e:
+ load_and_check_config(None)
+
+ assert e.value.args[0] == 'Configuration file must be defined'
+
+ config_path = '/indexer/inexistant/config.yml'
+ with pytest.raises(FileNotFoundError) as e:
+ load_and_check_config(config_path)
+
+ assert e.value.args[0] == 'Configuration file %s does not exist' % (
+ config_path, )
+
+
+def test_load_and_check_config_wrong_configuration(tmpdir):
+ """Wrong configuration raises"""
+ config_path = prepare_config_file(tmpdir, 'something: useless')
+ with pytest.raises(KeyError) as e:
+ load_and_check_config(config_path)
+
+ assert e.value.args[0] == 'Missing \'%indexer_storage\' configuration'
+
+
+def test_load_and_check_config_remote_config_local_type_raise(tmpdir):
+ """'local' configuration without 'local' storage raises"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'remote',
+ 'args': {}
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ with pytest.raises(EnvironmentError) as e:
+ load_and_check_config(config_path, type='local')
+
+ assert (
+ e.value.args[0] ==
+ "The indexer_storage backend can only be started with a 'local' "
+ "configuration"
+ )
+
+
+def test_load_and_check_config_local_incomplete_configuration(tmpdir):
+ """Incomplete 'local' configuration should raise"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'local',
+ 'args': {
+ }
+ }
+ }
+
+ config_path = prepare_config_file(tmpdir, config)
+ with pytest.raises(ValueError) as e:
+ load_and_check_config(config_path)
+
+ assert (
+ e.value.args[0] ==
+ "Invalid configuration; missing 'db' config entry"
+ )
+
+
+def test_load_and_check_config_local_config_fine(tmpdir):
+ """'Remote configuration is fine"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'local',
+ 'args': {
+ 'db': 'db',
+ }
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ cfg = load_and_check_config(config_path, type='local')
+ assert cfg == config
+
+
+def test_load_and_check_config_remote_config_fine(tmpdir):
+ """'Remote configuration is fine"""
+ config = {
+ 'indexer_storage': {
+ 'cls': 'remote',
+ 'args': {}
+ }
+ }
+ config_path = prepare_config_file(tmpdir, config)
+ cfg = load_and_check_config(config_path, type='any')
+
+ assert cfg == config

File Metadata

Mime Type
text/plain
Expires
Nov 5 2024, 5:57 AM (8 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217645

Event Timeline

About · Developer information · Legal