Page MenuHomeSoftware Heritage
Files F9342431

No OneTemporary
Actions

View Options

diff --git a/swh/search/api/server.py b/swh/search/api/server.py
index effd2ed..112c903 100644
--- a/swh/search/api/server.py
+++ b/swh/search/api/server.py
@@ -1,91 +1,100 @@
# Copyright (C) 2019-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import logging
import os
from typing import Any, Dict
from swh.core import config
from swh.core.api import RPCServerApp
from swh.core.api import encode_data_server as encode_data
from swh.core.api import error_handler
from swh.search.metrics import timed
from .. import get_search
from ..interface import SearchInterface
+logger = logging.getLogger(__name__)
+
def _get_search():
global search
if not search:
search = get_search(**app.config["search"])
return search
app = RPCServerApp(__name__, backend_class=SearchInterface, backend_factory=_get_search)
search = None
@app.errorhandler(Exception)
def my_error_handler(exception):
return error_handler(exception, encode_data)
@app.route("/")
@timed
def index():
return "SWH Search API server"
+@app.before_first_request
+def initialized_index():
+ search = _get_search()
+ logger.info("Initializing indexes with configuration: ", search.origin_config)
+ search.initialize()
+
+
api_cfg = None
def load_and_check_config(config_file: str) -> Dict[str, Any]:
"""Check the minimal configuration is set to run the api or raise an
error explanation.
Args:
config_file: Path to the configuration file to load
type: configuration type. For 'local' type, more
checks are done.
Raises:
Error if the setup is not as expected
Returns:
configuration as a dict
"""
if not config_file:
raise EnvironmentError("Configuration file must be defined")
if not os.path.exists(config_file):
raise FileNotFoundError("Configuration file %s does not exist" % (config_file,))
cfg = config.read(config_file)
if "search" not in cfg:
raise KeyError("Missing 'search' configuration")
return cfg
def make_app_from_configfile():
"""Run the WSGI app from the webserver, loading the configuration from
a configuration file.
SWH_CONFIG_FILENAME environment variable defines the
configuration path to load.
"""
global api_cfg
if not api_cfg:
config_file = os.environ.get("SWH_CONFIG_FILENAME")
api_cfg = load_and_check_config(config_file)
app.config.update(api_cfg)
handler = logging.StreamHandler()
app.logger.addHandler(handler)
return app
diff --git a/swh/search/tests/test_server.py b/swh/search/tests/test_server.py
index 6702beb..15b1b65 100644
--- a/swh/search/tests/test_server.py
+++ b/swh/search/tests/test_server.py
@@ -1,131 +1,156 @@
# Copyright (C) 2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information
import os
from typing import Any, Dict
import pytest
import yaml
from swh.core.api import RPCServerApp
from swh.core.config import load_from_envvar
+from swh.search.api import server
from swh.search.api.server import load_and_check_config, make_app_from_configfile
+def teardown_function():
+ # Ensure there is no configuration loaded from a previous test
+ server.api_cfg = None
+
+
def _write_config_file(tmp_path, monkeypatch, content):
conf_path = os.path.join(str(tmp_path), "search.yml")
with open(conf_path, "w") as f:
f.write(yaml.dump(content))
monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path)
return conf_path
@pytest.fixture
def swh_search_server_config_without_indexes() -> Dict[str, Any]:
return {"search": {"cls": "elasticsearch", "hosts": ["es1"],}}
@pytest.fixture
def swh_search_server_config_with_indexes(
swh_search_server_config_without_indexes,
) -> Dict[str, Any]:
return {
- **swh_search_server_config_without_indexes,
- **{"indexes": {"origin": {"index": "test"}}},
+ "search": {
+ **{"indexes": {"origin": {"index": "test"}}},
+ **swh_search_server_config_without_indexes["search"],
+ }
}
@pytest.fixture
def swh_search_config_without_indexes(
monkeypatch, swh_search_server_config_without_indexes, tmp_path
):
return _write_config_file(
tmp_path, monkeypatch, swh_search_server_config_without_indexes
)
@pytest.fixture
def swh_search_config_with_indexes(
monkeypatch, swh_search_server_config_with_indexes, tmp_path
):
return _write_config_file(
tmp_path, monkeypatch, swh_search_server_config_with_indexes
)
def prepare_config_file(tmpdir, config_dict: Dict, name: str = "config.yml") -> str:
"""Prepare configuration file in `$tmpdir/name` with content `content`.
Args:
tmpdir (LocalPath): root directory
content: Content of the file either as string or as a dict.
If a dict, converts the dict into a yaml string.
name: configuration filename
Returns
path of the configuration file prepared.
"""
config_path = tmpdir / name
config_path.write_text(yaml.dump(config_dict), encoding="utf-8")
# pytest on python3.5 does not support LocalPath manipulation, so
# convert path to string
return str(config_path)
@pytest.mark.parametrize("config_file", [None, ""])
def test_load_and_check_config_no_configuration(config_file):
"""Inexistent configuration files raises"""
with pytest.raises(EnvironmentError, match="Configuration file must be defined"):
load_and_check_config(config_file)
def test_load_and_check_config_inexistent_file():
config_path = "/some/inexistent/config.yml"
expected_error = f"Configuration file {config_path} does not exist"
with pytest.raises(EnvironmentError, match=expected_error):
load_and_check_config(config_path)
def test_load_and_check_config_wrong_configuration(tmpdir):
"""Wrong configuration raises"""
config_path = prepare_config_file(tmpdir, {"something": "useless"})
with pytest.raises(KeyError, match="Missing 'search' configuration"):
load_and_check_config(config_path)
def test_load_and_check_config_local_config_fine(
swh_search_server_config_with_indexes, tmpdir
):
"""'local' complete configuration is fine"""
config_path = prepare_config_file(tmpdir, swh_search_server_config_with_indexes)
cfg = load_and_check_config(config_path)
assert cfg == swh_search_server_config_with_indexes
def test_server_make_app_from_config_file_without_indexes(
swh_search_config_without_indexes,
):
app = make_app_from_configfile()
expected_cfg = load_from_envvar()
assert app is not None
assert isinstance(app, RPCServerApp)
assert app.config["search"] == expected_cfg["search"]
app2 = make_app_from_configfile()
assert app is app2
def test_server_make_app_from_config_file_with_indexes(swh_search_config_with_indexes,):
app = make_app_from_configfile()
expected_cfg = load_from_envvar()
-
assert app is not None
assert isinstance(app, RPCServerApp)
assert app.config["search"] == expected_cfg["search"]
app2 = make_app_from_configfile()
assert app is app2
+
+
+def test_server_first_call_initialize_elasticsearch(
+ swh_search_config_with_indexes, mocker
+):
+ """Test the initialize method is called during the first and first only
+ request to the server"""
+ mock = mocker.patch("swh.search.elasticsearch.ElasticSearch.initialize")
+
+ app = make_app_from_configfile()
+ app.config["TESTING"] = True
+ tc = app.test_client()
+
+ tc.get("/")
+ assert mock.call_count == 1
+
+ tc.get("/")
+ assert mock.call_count == 1

File Metadata

Mime Type
text/x-diff
Expires
Fri, Jul 4, 12:43 PM (2 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3293405

Event Timeline

About · Developer information · Legal