diff --git a/requirements-swh.txt b/requirements-swh.txt --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,5 +1,6 @@ swh.auth[django] >= 0.5.0 swh.core >= 0.0.95 +swh.counters >= 0.5.1 swh.indexer >= 0.4.1 swh.model >= 0.5.0 swh.scheduler >= 0.7.0 diff --git a/swh/web/common/archive.py b/swh/web/common/archive.py --- a/swh/web/common/archive.py +++ b/swh/web/common/archive.py @@ -31,6 +31,7 @@ storage = config.storage() vault = config.vault() idx_storage = config.indexer_storage() +counters = config.counters() MAX_LIMIT = 50 # Top limit the users can ask for @@ -923,7 +924,14 @@ Returns: A dict mapping textual labels to integer values. """ - return storage.stat_counters() + res = {} + if counters and config.get_config()["counters_backend"] == "swh-counters": + res = counters.get_counts( + ["origin", "revision", "content", "directory", "release"] + ) + else: + res = storage.stat_counters() + return res def _lookup_origin_visits( diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -7,6 +7,7 @@ from typing import Any, Dict from swh.core import config +from swh.counters import get_counters from swh.indexer.storage import get_indexer_storage from swh.scheduler import get_scheduler from swh.search import get_search @@ -52,6 +53,10 @@ "dict", {"cls": "remote", "url": "http://127.0.0.1:5007/", "timeout": 1,}, ), + "counters": ( + "dict", + {"cls": "remote", "url": "http://127.0.0.1:5011/", "timeout": 1,}, + ), "log_dir": ("string", "/tmp/swh/log"), "debug": ("bool", False), "serve_assets": ("bool", False), @@ -124,6 +129,7 @@ }, ), "metadata_search_backend": ("string", "swh-indexer-storage"), # or "swh-search" + "counters_backend": ("string", "swh-storage"), # or "swh-counters" "staging_server_names": ("list", STAGING_SERVER_NAMES), } @@ -161,6 +167,7 @@ **swhweb_config["indexer_storage"] ) swhweb_config["scheduler"] = get_scheduler(**swhweb_config["scheduler"]) + swhweb_config["counters"] = get_counters(**swhweb_config["counters"]) return swhweb_config @@ -197,3 +204,10 @@ """ return get_config()["scheduler"] + + +def counters(): + """Return the current application's counters. + + """ + return get_config()["counters"] diff --git a/swh/web/settings/tests.py b/swh/web/settings/tests.py --- a/swh/web/settings/tests.py +++ b/swh/web/settings/tests.py @@ -100,7 +100,10 @@ test_data = get_tests_data() override_storages( - test_data["storage"], test_data["idx_storage"], test_data["search"] + test_data["storage"], + test_data["idx_storage"], + test_data["search"], + test_data["counters"], ) else: ALLOWED_HOSTS += ["testserver", SWH_WEB_INTERNAL_SERVER_NAME] diff --git a/swh/web/tests/api/views/test_origin.py b/swh/web/tests/api/views/test_origin.py --- a/swh/web/tests/api/views/test_origin.py +++ b/swh/web/tests/api/views/test_origin.py @@ -689,3 +689,18 @@ url = reverse("api-1-origin-metadata-search") check_api_get_responses(api_client, url, status_code=400) mock_idx_storage.assert_not_called() + + +@pytest.mark.parametrize("backend", ["swh-counters", "swh-storage"]) +def test_api_stat_counters(api_client, mocker, backend): + + mock_config = mocker.patch("swh.web.common.archive.config") + mock_config.get_config.return_value = {"counters_backend": backend} + + url = reverse("api-1-stat-counters") + rv = check_api_get_responses(api_client, url, status_code=200) + + counts = json.loads(rv.content) + + for obj in ["content", "origin", "release", "directory", "revision"]: + assert counts.get(obj, 0) > 0 diff --git a/swh/web/tests/conftest.py b/swh/web/tests/conftest.py --- a/swh/web/tests/conftest.py +++ b/swh/web/tests/conftest.py @@ -142,7 +142,9 @@ data = get_tests_data(reset=True) # Update swh-web configuration to use the in-memory storages # instantiated in the tests.data module - override_storages(data["storage"], data["idx_storage"], data["search"]) + override_storages( + data["storage"], data["idx_storage"], data["search"], data["counters"] + ) return data diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -11,6 +11,7 @@ from typing import Dict, List, Optional, Set from swh.core.config import merge_configs +from swh.counters import get_counters from swh.indexer.ctags import CtagsIndexer from swh.indexer.fossology_license import FossologyLicenseIndexer from swh.indexer.mimetype import MimetypeIndexer @@ -173,16 +174,21 @@ ORIGIN_MASTER_REVISION = {} -def _add_origin(storage, search, origin_url, visit_type="git", snapshot_branches={}): +def _add_origin( + storage, search, counters, origin_url, visit_type="git", snapshot_branches={} +): storage.origin_add([Origin(url=origin_url)]) search.origin_update( [{"url": origin_url, "has_visits": True, "visit_types": [visit_type]}] ) + counters.add("origin", [origin_url]) date = now() visit = OriginVisit(origin=origin_url, date=date, type=visit_type) visit = storage.origin_visit_add([visit])[0] + counters.add("origin_visit", [f"{visit.unique_key()}"]) snapshot = Snapshot.from_dict({"branches": snapshot_branches}) storage.snapshot_add([snapshot]) + counters.add("snapshot", [snapshot.id]) visit_status = OriginVisitStatus( origin=origin_url, visit=visit.visit, @@ -192,6 +198,7 @@ snapshot=snapshot.id, ) storage.origin_visit_status_add([visit_status]) + counters.add("origin_visit_status", [f"{visit_status.unique_key()}"]) # Tests data initialization @@ -204,6 +211,9 @@ search.initialize() search.origin_update({"url": origin["url"]} for origin in _TEST_ORIGINS) + # create the counters instance + counters = get_counters("memory") + # Create indexer storage instance that will be shared by indexers idx_storage = get_indexer_storage("memory") @@ -236,7 +246,11 @@ for i in range(250): _add_origin( - storage, search, origin_url=f"https://many.origins/{i+1}", visit_type="tar" + storage, + search, + counters, + origin_url=f"https://many.origins/{i+1}", + visit_type="tar", ) sha1s: Set[Sha1] = set() @@ -343,6 +357,7 @@ _add_origin( storage, search, + counters, origin_url="https://git.example.org/project", snapshot_branches={ b"refs/heads/master": { @@ -359,11 +374,17 @@ }, ) + counters.add("revision", revisions) + counters.add("release", releases) + counters.add("directory", directories) + counters.add("content", [content["sha1"] for content in contents]) + # Return tests data return { "search": search, "storage": storage, "idx_storage": idx_storage, + "counters": counters, "origins": _TEST_ORIGINS, "contents": contents, "directories": list(directories), @@ -424,16 +445,22 @@ return _current_tests_data -def override_storages(storage, idx_storage, search): +def override_storages(storage, idx_storage, search, counters): """ Helper function to replace the storages from which archive data are fetched. """ swh_config = config.get_config() swh_config.update( - {"storage": storage, "indexer_storage": idx_storage, "search": search,} + { + "storage": storage, + "indexer_storage": idx_storage, + "search": search, + "counters": counters, + } ) archive.storage = storage archive.idx_storage = idx_storage archive.search = search + archive.counters = counters