diff --git a/swh/counters/api/server.py b/swh/counters/api/server.py --- a/swh/counters/api/server.py +++ b/swh/counters/api/server.py @@ -29,7 +29,10 @@ handler = logging.StreamHandler() app.logger.addHandler(handler) + app.config["counters"] = get_counters(**config["counters"]) + app.add_url_rule("/", "index", index) + app.add_url_rule("/metrics", "metrics", get_metrics) return app @@ -84,3 +87,32 @@ app = make_app(api_cfg) return app + + +def get_metrics(): + """expose the counters values in a prometheus format + + detailed format: + # HELP swh_archive_object_total Software Heritage Archive object counters + # TYPE swh_archive_object_total gauge + swh_archive_object_total{col="value",object_type=""} + ... + """ + + response = [ + "# HELP swh_archive_object_total Software Heritage Archive object counters", + "# TYPE swh_archive_object_total gauge", + ] + counters = app.config["counters"] + + for collection in counters.get_counters(): + collection_name = collection.decode("utf-8") + value = counters.get_count(collection) + line = 'swh_archive_object_total{col="value", object_type="%s"} %s' % ( + collection_name, + value, + ) + response.append(line) + response.append("") + + return "\n".join(response) diff --git a/swh/counters/interface.py b/swh/counters/interface.py --- a/swh/counters/interface.py +++ b/swh/counters/interface.py @@ -26,3 +26,7 @@ def get_count(self, collection: str) -> int: """Return the number of keys for the provided collection""" ... + + @remote_api_endpoint("counters") + def get_counters(self) -> Iterable[str]: + """Return the list of managed counters""" diff --git a/swh/counters/redis.py b/swh/counters/redis.py --- a/swh/counters/redis.py +++ b/swh/counters/redis.py @@ -55,3 +55,6 @@ def get_count(self, collection: str) -> int: return self.redis_client.pfcount(collection) + + def get_counters(self) -> Iterable[str]: + return self.redis_client.keys() diff --git a/swh/counters/tests/conftest.py b/swh/counters/tests/conftest.py --- a/swh/counters/tests/conftest.py +++ b/swh/counters/tests/conftest.py @@ -6,6 +6,7 @@ import logging import pytest +from redis import Redis as RedisClient logger = logging.getLogger(__name__) @@ -24,3 +25,11 @@ return JOURNAL_OBJECTS_CONFIG_TEMPLATE.format( broker=kafka_server, group_id="test-consumer", prefix=kafka_prefix ) + + +@pytest.fixture +def local_redis(redis_proc): + yield redis_proc + # Cleanup redis between 2 tests + rc = RedisClient(host=redis_proc.host, port=redis_proc.port) + rc.flushall() diff --git a/swh/counters/tests/test_redis.py b/swh/counters/tests/test_redis.py --- a/swh/counters/tests/test_redis.py +++ b/swh/counters/tests/test_redis.py @@ -4,12 +4,10 @@ # See top-level LICENSE file for more information import pytest -from pytest_redis import factories +from redis import Redis as RedisClient from swh.counters.redis import DEFAULT_REDIS_PORT, Redis -local_redis = factories.redis_proc(host="localhost") - def test__redis__constructor(): r = Redis("fakehost") @@ -66,3 +64,17 @@ assert 1 == r.get_count("c2") assert 2 == r.get_count("c3") assert 0 == r.get_count("c4") + + +def test__redis__collections(local_redis): + client = RedisClient(host=local_redis.host, port=local_redis.port) + client.pfadd("counter1", b"k1") + client.pfadd("counter2", b"k2") + + r = Redis("%s:%d" % (local_redis.host, local_redis.port)) + + counters = r.get_counters() + + assert 2 == len(counters) + assert b"counter1" in counters + assert b"counter2" in counters diff --git a/swh/counters/tests/test_server.py b/swh/counters/tests/test_server.py --- a/swh/counters/tests/test_server.py +++ b/swh/counters/tests/test_server.py @@ -3,9 +3,11 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import re from typing import Any, Dict import pytest +from redis import Redis as RedisClient import yaml from swh.core.api import RPCServerApp @@ -15,12 +17,12 @@ def teardown_function(): # Ensure there is no configuration loaded from a previous test - server.api = None + server.app = None @pytest.fixture def swh_counters_server_config() -> Dict[str, Any]: - return {"counters": {"cls": "redis", "hosts": "redis",}} + return {"counters": {"cls": "redis", "host": "redis",}} @pytest.fixture @@ -97,3 +99,47 @@ assert 200 == r.status_code assert b"SWH Counters" in r.get_data() + + +def test_server_metrics(local_redis, tmp_path, monkeypatch): + """Test the metrics generation""" + + rc = RedisClient(host=local_redis.host, port=local_redis.port) + data = { + "col1": 1, + "col2": 4, + "col3": 6, + "col4": 10, + } + + for coll in data.keys(): + for i in range(0, data[coll]): + rc.pfadd(coll, i) + + cfg = { + "counters": {"cls": "redis", "host": f"{local_redis.host}:{local_redis.port}"} + } + _environment_config_file(tmp_path, monkeypatch, cfg) + + app = make_app_from_configfile() + app.config["TESTING"] = True + tc = app.test_client() + + r = tc.get("/metrics") + + assert 200 == r.status_code + + response = r.get_data().decode("utf-8") + + assert "HELP" in response + assert "TYPE" in response + + for collection in data.keys(): + obj_type = f'object_type="{collection}"' + assert obj_type in response + + pattern = r'swh_archive_object_total{col="value", object_type="%s"} (\d+)' % ( + collection + ) + m = re.search(pattern, response) + assert data[collection] == int(m.group(1))