diff --git a/swh/search/tests/test_api_client.py b/swh/search/tests/test_api_client.py --- a/swh/search/tests/test_api_client.py +++ b/swh/search/tests/test_api_client.py @@ -11,10 +11,12 @@ from swh.search import get_search from swh.search.api.server import app -from .test_search import CommonSearchTest +from .test_elasticsearch import CommonElasticsearchSearchTest -class TestRemoteSearch(CommonSearchTest, ServerTestFixture, unittest.TestCase): +class TestRemoteSearch( + CommonElasticsearchSearchTest, ServerTestFixture, unittest.TestCase +): @pytest.fixture(autouse=True) def _instantiate_search(self, elasticsearch_host): self._elasticsearch_host = elasticsearch_host diff --git a/swh/search/tests/test_elasticsearch.py b/swh/search/tests/test_elasticsearch.py --- a/swh/search/tests/test_elasticsearch.py +++ b/swh/search/tests/test_elasticsearch.py @@ -48,108 +48,9 @@ ] -class BaseElasticsearchTest(unittest.TestCase): - @pytest.fixture(autouse=True) - def _instantiate_search(self, swh_search, elasticsearch_host, mocker): - self._elasticsearch_host = elasticsearch_host - self.search = swh_search - self.mocker = mocker - - # override self.search.origin_update to catch painless script errors - # and pretty print them - origin_update = self.search.origin_update - - def _origin_update(self, *args, **kwargs): - script_error = False - error_detail = "" - try: - origin_update(*args, **kwargs) - except BulkIndexError as e: - error = e.errors[0].get("update", {}).get("error", {}).get("caused_by") - if error and "script_stack" in error: - script_error = True - error_detail = dedent( - f""" - Painless update script failed ({error.get('reason')}). - error type: {error.get('caused_by', {}).get('type')} - error reason: {error.get('caused_by', {}).get('reason')} - script stack: - - """ - ) - error_detail += "\n".join(error["script_stack"]) - else: - raise e - assert script_error is False, error_detail[1:] - - self.search.origin_update = types.MethodType(_origin_update, self.search) - - def reset(self): - self.search.deinitialize() - self.search.initialize() - - -class TestElasticsearchSearch(CommonSearchTest, BaseElasticsearchTest): - def test_metrics_update_duration(self): - mock = self.mocker.patch("swh.search.metrics.statsd.timing") - - for url in ["http://foobar.bar", "http://foobar.baz"]: - self.search.origin_update([{"url": url}]) - - assert mock.call_count == 2 - - def test_metrics_search_duration(self): - mock = self.mocker.patch("swh.search.metrics.statsd.timing") - - for url_pattern in ["foobar", "foobaz"]: - self.search.origin_search(url_pattern=url_pattern, with_visit=True) - - assert mock.call_count == 2 - - def test_metrics_indexation_counters(self): - mock_es = self.mocker.patch("elasticsearch.helpers.bulk") - mock_es.return_value = 2, ["error"] - - mock_metrics = self.mocker.patch("swh.search.metrics.statsd.increment") - - self.search.origin_update([{"url": "http://foobar.baz"}]) - - assert mock_metrics.call_count == 2 - - mock_metrics.assert_any_call( - OPERATIONS_METRIC, - 2, - tags={ - "endpoint": "origin_update", - "object_type": "document", - "operation": "index", - }, - ) - mock_metrics.assert_any_call( - OPERATIONS_METRIC, - 1, - tags={ - "endpoint": "origin_update", - "object_type": "document", - "operation": "index_error", - }, - ) - - def test_write_alias_usage(self): - mock = self.mocker.patch("elasticsearch.helpers.bulk") - mock.return_value = 2, ["result"] - - self.search.origin_update([{"url": "http://foobar.baz"}]) - - assert mock.call_args[1]["index"] == "test-write" - - def test_read_alias_usage(self): - mock = self.mocker.patch("elasticsearch.Elasticsearch.search") - mock.return_value = {"hits": {"hits": []}} - - self.search.origin_search(url_pattern="foobar.baz") - - assert mock.call_args[1]["index"] == "test-read" +class CommonElasticsearchSearchTest(CommonSearchTest): + """Tests shared between this module (direct ES backend test) and test_api_client.py + (ES backend via HTTP test)""" def test_sort_by_and_limit_query(self): @@ -276,3 +177,105 @@ with pytest.raises(SearchQuerySyntaxError): self.search.origin_search(query="foobar") + + +class TestElasticsearchSearch(CommonElasticsearchSearchTest, unittest.TestCase): + @pytest.fixture(autouse=True) + def _instantiate_search(self, swh_search, elasticsearch_host, mocker): + self._elasticsearch_host = elasticsearch_host + self.search = swh_search + self.mocker = mocker + + # override self.search.origin_update to catch painless script errors + # and pretty print them + origin_update = self.search.origin_update + + def _origin_update(self, *args, **kwargs): + script_error = False + error_detail = "" + try: + origin_update(*args, **kwargs) + except BulkIndexError as e: + error = e.errors[0].get("update", {}).get("error", {}).get("caused_by") + if error and "script_stack" in error: + script_error = True + error_detail = dedent( + f""" + Painless update script failed ({error.get('reason')}). + error type: {error.get('caused_by', {}).get('type')} + error reason: {error.get('caused_by', {}).get('reason')} + script stack: + + """ + ) + error_detail += "\n".join(error["script_stack"]) + else: + raise e + assert script_error is False, error_detail[1:] + + self.search.origin_update = types.MethodType(_origin_update, self.search) + + def reset(self): + self.search.deinitialize() + self.search.initialize() + + def test_metrics_update_duration(self): + mock = self.mocker.patch("swh.search.metrics.statsd.timing") + + for url in ["http://foobar.bar", "http://foobar.baz"]: + self.search.origin_update([{"url": url}]) + + assert mock.call_count == 2 + + def test_metrics_search_duration(self): + mock = self.mocker.patch("swh.search.metrics.statsd.timing") + + for url_pattern in ["foobar", "foobaz"]: + self.search.origin_search(url_pattern=url_pattern, with_visit=True) + + assert mock.call_count == 2 + + def test_metrics_indexation_counters(self): + mock_es = self.mocker.patch("elasticsearch.helpers.bulk") + mock_es.return_value = 2, ["error"] + + mock_metrics = self.mocker.patch("swh.search.metrics.statsd.increment") + + self.search.origin_update([{"url": "http://foobar.baz"}]) + + assert mock_metrics.call_count == 2 + + mock_metrics.assert_any_call( + OPERATIONS_METRIC, + 2, + tags={ + "endpoint": "origin_update", + "object_type": "document", + "operation": "index", + }, + ) + mock_metrics.assert_any_call( + OPERATIONS_METRIC, + 1, + tags={ + "endpoint": "origin_update", + "object_type": "document", + "operation": "index_error", + }, + ) + + def test_write_alias_usage(self): + mock = self.mocker.patch("elasticsearch.helpers.bulk") + mock.return_value = 2, ["result"] + + self.search.origin_update([{"url": "http://foobar.baz"}]) + + assert mock.call_args[1]["index"] == "test-write" + + def test_read_alias_usage(self): + mock = self.mocker.patch("elasticsearch.Elasticsearch.search") + mock.return_value = {"hits": {"hits": []}} + + self.search.origin_search(url_pattern="foobar.baz") + + assert mock.call_args[1]["index"] == "test-read"