diff --git a/PKG-INFO b/PKG-INFO index 303c304..06b05d9 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,29 +1,29 @@ Metadata-Version: 2.1 Name: swh.counters -Version: 0.4.0 +Version: 0.4.1 Summary: Software Heritage Next gen counters Home-page: https://forge.softwareheritage.org/source/swh-counters Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-counters Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-counters/ Description: .. _swh-counters: swh-counters ============ Service providing efficient estimates of the number of objects in the SWH archive, using Redis's Hyperloglog Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/debian/changelog b/debian/changelog index b492b74..4b06f9b 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,65 +1,67 @@ -swh.counters (0.4.0-1~swh1~bpo10+1) buster-swh; urgency=medium +swh.counters (0.4.1-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 0.4.1 - (tagged by Vincent SELLIER + on 2021-04-08 12:20:16 +0200) + * Upstream changes: - v0.4.1 - * fix debian stable build - -- Software Heritage autobuilder (on jenkins-debian1) Wed, 07 Apr 2021 15:15:15 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Thu, 08 Apr 2021 10:22:40 +0000 swh.counters (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Vincent SELLIER on 2021-04-07 17:09:57 +0200) * Upstream changes: - v0.4.0 - Add a GET endpoint to retrieve the historical data - Use a intermediate temporary file during the generation - of the historical data file -- Software Heritage autobuilder (on jenkins-debian1) Wed, 07 Apr 2021 15:14:08 +0000 swh.counters (0.3.0-1~swh2) unstable-swh; urgency=medium * Fix build dependency -- Vincent SELLIER Wed, 07 Apr 2021 12:37:36 +0200 swh.counters (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by Vincent SELLIER on 2021-04-07 12:13:40 +0200) * Upstream changes: - v0.3.0 - improve the documentation - add management of the counters's historical data -- Software Heritage autobuilder (on jenkins-debian1) Wed, 07 Apr 2021 10:17:30 +0000 swh.counters (0.2.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.1 - (tagged by Valentin Lorentz on 2021-04-06 17:24:25 +0200) * Upstream changes: - v0.2.1 - Documentation improvements -- Software Heritage autobuilder (on jenkins-debian1) Tue, 06 Apr 2021 15:26:14 +0000 swh.counters (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Vincent SELLIER on 2021-03-25 10:15:51 +0100) * Upstream changes: - v0.2.0 - Expose counters as prometheus metrics -- Software Heritage autobuilder (on jenkins-debian1) Thu, 25 Mar 2021 09:18:18 +0000 swh.counters (0.1.0-1+swh3) unstable-swh; urgency=medium * Rebuild to remove the swh/__init__.py file from the package -- Vincent SELLIER Fri, 19 Mar 2021 17:25:15 +0100 swh.counters (0.1.0-1+swh2) unstable-swh; urgency=medium * Rebuild to unstuck the debian packaging. -- Vincent Sellier Wed, 17 Mar 2021 17:20:49 +0100 swh.counters (0.1.0-1+swh1) unstable-swh; urgency=medium * Initial release. -- Vincent Sellier Wed, 17 Mar 2021 17:20:49 +0100 diff --git a/swh.counters.egg-info/PKG-INFO b/swh.counters.egg-info/PKG-INFO index 303c304..06b05d9 100644 --- a/swh.counters.egg-info/PKG-INFO +++ b/swh.counters.egg-info/PKG-INFO @@ -1,29 +1,29 @@ Metadata-Version: 2.1 Name: swh.counters -Version: 0.4.0 +Version: 0.4.1 Summary: Software Heritage Next gen counters Home-page: https://forge.softwareheritage.org/source/swh-counters Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-counters Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-counters/ Description: .. _swh-counters: swh-counters ============ Service providing efficient estimates of the number of objects in the SWH archive, using Redis's Hyperloglog Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 3 - Alpha Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh/counters/tests/test_server.py b/swh/counters/tests/test_server.py index 6bba4f1..e938b5d 100644 --- a/swh/counters/tests/test_server.py +++ b/swh/counters/tests/test_server.py @@ -1,193 +1,193 @@ # Copyright (C) 2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json import re from typing import Any, Dict import pytest from redis import Redis as RedisClient import yaml from swh.core.api import RPCServerApp from swh.counters.api import server from swh.counters.api.server import load_and_check_config, make_app_from_configfile def teardown_function(): # Ensure there is no configuration loaded from a previous test server.app = None @pytest.fixture def swh_counters_server_config() -> Dict[str, Any]: return {"counters": {"cls": "redis", "host": "redis",}} @pytest.fixture def swh_counters_server_config_on_disk( tmp_path, monkeypatch, swh_counters_server_config ) -> str: return _environment_config_file(tmp_path, monkeypatch, swh_counters_server_config) @pytest.fixture def history_test_client(tmp_path, monkeypatch): cfg = { "counters": {"cls": "redis", "host": "redis:6379"}, "history": { "cls": "prometheus", "prometheus_host": "prometheus", "prometheus_port": "9090", "live_data_start": "0", "cache_base_directory": "/tmp", }, } _environment_config_file(tmp_path, monkeypatch, cfg) app = make_app_from_configfile() app.config["TESTING"] = True return app.test_client() def write_config_file(tmpdir, config_dict: Dict, name: str = "config.yml") -> str: """Prepare configuration file in `$tmpdir/name` with content `content`. Args: tmpdir (LocalPath): root directory content: Content of the file either as string or as a dict. If a dict, converts the dict into a yaml string. name: configuration filename Returns path of the configuration file prepared. """ config_path = tmpdir / name config_path.write_text(yaml.dump(config_dict), encoding="utf-8") # pytest on python3.5 does not support LocalPath manipulation, so # convert path to string return str(config_path) def _environment_config_file(tmp_path, monkeypatch, content): conf_path = write_config_file(tmp_path, content) monkeypatch.setenv("SWH_CONFIG_FILENAME", conf_path) @pytest.mark.parametrize("config_file", [None, ""]) def test_load_and_check_config_no_configuration(config_file): """Inexistent configuration files raises""" with pytest.raises(EnvironmentError, match="Configuration file must be defined"): load_and_check_config(config_file) def test_load_and_check_config_inexistent_file(): config_path = "/some/inexistent/config.yml" expected_error = f"Configuration file {config_path} does not exist" with pytest.raises(EnvironmentError, match=expected_error): load_and_check_config(config_path) def test_load_and_check_config_wrong_configuration(tmpdir): """Wrong configuration raises""" config_path = write_config_file(tmpdir, {"something": "useless"}) with pytest.raises(KeyError, match="Missing 'counters' configuration"): load_and_check_config(config_path) def test_server_make_app_from_config_file(swh_counters_server_config_on_disk): app = make_app_from_configfile() assert app is not None assert isinstance(app, RPCServerApp) app2 = make_app_from_configfile() assert app is app2 def test_server_index(swh_counters_server_config_on_disk, mocker): """Test the result of the main page""" app = make_app_from_configfile() app.config["TESTING"] = True tc = app.test_client() r = tc.get("/") assert 200 == r.status_code assert b"SWH Counters" in r.get_data() def test_server_metrics(local_redis, tmp_path, monkeypatch): """Test the metrics generation""" rc = RedisClient(host=local_redis.host, port=local_redis.port) data = { "col1": 1, "col2": 4, "col3": 6, "col4": 10, } for coll in data.keys(): for i in range(0, data[coll]): rc.pfadd(coll, i) cfg = { "counters": {"cls": "redis", "host": f"{local_redis.host}:{local_redis.port}"} } _environment_config_file(tmp_path, monkeypatch, cfg) app = make_app_from_configfile() app.config["TESTING"] = True tc = app.test_client() r = tc.get("/metrics") assert 200 == r.status_code response = r.get_data().decode("utf-8") assert "HELP" in response assert "TYPE" in response for collection in data.keys(): obj_type = f'object_type="{collection}"' assert obj_type in response pattern = r'swh_archive_object_total{col="value", object_type="%s"} (\d+)' % ( collection ) m = re.search(pattern, response) assert data[collection] == int(m.group(1)) def test_server_counters_history(history_test_client, mocker): """Test the counters history file download""" expected_result = {"content": [[1, 1], [2, 2]]} mock = mocker.patch("swh.counters.history.History.get_history") - mock.return_value = expected_result + mock.return_value = json.dumps(expected_result) r = history_test_client.get("/counters_history/test.json") assert 200 == r.status_code response = r.get_data().decode("utf-8") response_json = json.loads(response) assert response_json == expected_result def test_server_counters_history_file_not_found(history_test_client, mocker): """ensure a 404 is returned when the file doesn't exists""" mock = mocker.patch("swh.counters.history.History.get_history") mock.side_effect = FileNotFoundError r = history_test_client.get("/counters_history/fake.json") assert 404 == r.status_code