diff --git a/mypy.ini b/mypy.ini --- a/mypy.ini +++ b/mypy.ini @@ -27,6 +27,9 @@ [mypy-pkg_resources.*] ignore_missing_imports = True +[mypy-prometheus_client.*] +ignore_missing_imports = True + [mypy-pygments.*] ignore_missing_imports = True diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ python-magic >= 0.4.0 htmlmin lxml +prometheus_client pygments pypandoc python-dateutil diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -5,6 +5,7 @@ from bisect import bisect_right from datetime import datetime, timezone, timedelta +from itertools import product import json import logging @@ -13,6 +14,8 @@ from django.core.validators import URLValidator from django.utils.html import escape +from prometheus_client import Gauge + import requests import sentry_sdk @@ -23,10 +26,11 @@ SaveUnauthorizedOrigin, SaveAuthorizedOrigin, SaveOriginRequest, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, - SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING + SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING, + SAVE_TASK_NOT_CREATED ) from swh.web.common.origin_visits import get_origin_visits -from swh.web.common.utils import parse_timestamp +from swh.web.common.utils import parse_timestamp, SWH_WEB_METRICS_REGISTRY from swh.scheduler.utils import create_oneshot_task_dict @@ -527,3 +531,54 @@ sentry_sdk.capture_exception(exc) return task_run + + +SUBMITTED_SAVE_REQUESTS_METRIC = 'swh_web_submitted_save_requests' + +_submitted_save_requests_gauge = Gauge( + name=SUBMITTED_SAVE_REQUESTS_METRIC, + documentation='Number of submitted origin save requests', + labelnames=['status', 'visit_type'], + registry=SWH_WEB_METRICS_REGISTRY) + + +ACCEPTED_SAVE_REQUESTS_METRIC = 'swh_web_accepted_save_requests' + +_accepted_save_requests_gauge = Gauge( + name=ACCEPTED_SAVE_REQUESTS_METRIC, + documentation='Number of accepted origin save requests', + labelnames=['load_task_status', 'visit_type'], + registry=SWH_WEB_METRICS_REGISTRY) + + +def compute_save_requests_metrics(): + """Compute a couple of Prometheus metrics related to + origin save requests""" + + request_statuses = (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING) + + load_task_statuses = (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, + SAVE_TASK_FAILED, SAVE_TASK_RUNNING) + + visit_types = get_savable_visit_types() + + labels_set = product(request_statuses, visit_types) + + for labels in labels_set: + _submitted_save_requests_gauge.labels(*labels).set(0) + + labels_set = product(load_task_statuses, visit_types) + + for labels in labels_set: + _accepted_save_requests_gauge.labels(*labels).set(0) + + for sor in SaveOriginRequest.objects.all(): + if sor.status == SAVE_REQUEST_ACCEPTED: + _accepted_save_requests_gauge.labels( + load_task_status=sor.loading_task_status, + visit_type=sor.visit_type).inc() + + _submitted_save_requests_gauge.labels( + status=sor.status, visit_type=sor.visit_type).inc() diff --git a/swh/web/common/utils.py b/swh/web/common/utils.py --- a/swh/web/common/utils.py +++ b/swh/web/common/utils.py @@ -14,6 +14,8 @@ from django.urls import reverse as django_reverse from django.http import QueryDict +from prometheus_client.registry import CollectorRegistry + from rest_framework.authentication import SessionAuthentication from swh.model.exceptions import ValidationError @@ -24,6 +26,8 @@ from swh.web.common.exc import BadInputExc +SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) + swh_object_icons = { 'branch': 'fa fa-code-fork', 'branches': 'fa fa-code-fork', diff --git a/swh/web/misc/metrics.py b/swh/web/misc/metrics.py new file mode 100644 --- /dev/null +++ b/swh/web/misc/metrics.py @@ -0,0 +1,19 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.http import HttpResponse + +from prometheus_client.exposition import generate_latest, CONTENT_TYPE_LATEST +from swh.web.common.origin_save import compute_save_requests_metrics +from swh.web.common.utils import SWH_WEB_METRICS_REGISTRY + + +def prometheus_metrics(request): + + compute_save_requests_metrics() + + return HttpResponse( + content=generate_latest(registry=SWH_WEB_METRICS_REGISTRY), + content_type=CONTENT_TYPE_LATEST) diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -15,6 +15,7 @@ from swh.web.common import service from swh.web.config import get_config +from swh.web.misc.metrics import prometheus_metrics def _jslicenses(request): @@ -45,8 +46,10 @@ url(r'^', include('swh.web.misc.coverage')), url(r'^jslicenses/$', _jslicenses, name='jslicenses'), url(r'^', include('swh.web.misc.origin_save')), - url(r'^stat_counters', _stat_counters, name='stat-counters'), + url(r'^stat_counters/', _stat_counters, name='stat-counters'), url(r'^', include('swh.web.misc.badges')), + url(r'^metrics/prometheus/$', prometheus_metrics, + name='metrics-prometheus'), ] diff --git a/swh/web/tests/misc/test_metrics.py b/swh/web/tests/misc/test_metrics.py new file mode 100644 --- /dev/null +++ b/swh/web/tests/misc/test_metrics.py @@ -0,0 +1,79 @@ +# Copyright (C) 2019 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from itertools import product +import random + +from prometheus_client.exposition import CONTENT_TYPE_LATEST + +import pytest + +from swh.web.common.models import ( + SaveOriginRequest, + SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_REQUEST_PENDING, + SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_SCHEDULED, + SAVE_TASK_SUCCEED, SAVE_TASK_FAILED, SAVE_TASK_RUNNING, + SAVE_TASK_NOT_CREATED +) +from swh.web.common.origin_save import ( + get_savable_visit_types, ACCEPTED_SAVE_REQUESTS_METRIC, + SUBMITTED_SAVE_REQUESTS_METRIC +) +from swh.web.common.utils import reverse +from swh.web.tests.django_asserts import assert_contains + + +@pytest.mark.django_db +def test_origin_save_metrics(client): + visit_types = get_savable_visit_types() + request_statuses = (SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING) + + load_task_statuses = (SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, SAVE_TASK_SUCCEED, + SAVE_TASK_FAILED, SAVE_TASK_RUNNING) + + for _ in range(random.randint(50, 100)): + visit_type = random.choice(visit_types) + request_satus = random.choice(request_statuses) + load_task_status = random.choice(load_task_statuses) + + SaveOriginRequest.objects.create(origin_url='origin', + visit_type=visit_type, + status=request_satus, + loading_task_status=load_task_status) + + url = reverse('metrics-prometheus') + resp = client.get(url) + + assert resp.status_code == 200 + assert resp['Content-Type'] == CONTENT_TYPE_LATEST + + accepted_requests = SaveOriginRequest.objects.filter( + status=SAVE_REQUEST_ACCEPTED) + + labels_set = product(visit_types, load_task_statuses) + + for labels in labels_set: + sor_count = accepted_requests.filter( + visit_type=labels[0], loading_task_status=labels[1]).count() + + metric_text = (f'{ACCEPTED_SAVE_REQUESTS_METRIC}{{' + f'load_task_status="{labels[1]}",' + f'visit_type="{labels[0]}"}} {float(sor_count)}\n') + + assert_contains(resp, metric_text) + + labels_set = product(visit_types, request_statuses) + + for labels in labels_set: + sor_count = SaveOriginRequest.objects.filter( + visit_type=labels[0], status=labels[1]).count() + + metric_text = (f'{SUBMITTED_SAVE_REQUESTS_METRIC}{{' + f'status="{labels[1]}",' + f'visit_type="{labels[0]}"}} {float(sor_count)}\n') + + assert_contains(resp, metric_text)