diff --git a/swh/web/config.py b/swh/web/config.py --- a/swh/web/config.py +++ b/swh/web/config.py @@ -165,6 +165,7 @@ "swh.web.deposit", "swh.web.badges", "swh.web.archive_coverage", + "swh.web.metrics", ], ), } diff --git a/swh/web/metrics/__init__.py b/swh/web/metrics/__init__.py new file mode 100644 diff --git a/swh/web/metrics/prometheus.py b/swh/web/metrics/prometheus.py new file mode 100644 --- /dev/null +++ b/swh/web/metrics/prometheus.py @@ -0,0 +1,124 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from itertools import product + +from prometheus_client import Gauge +from prometheus_client.registry import CollectorRegistry + +from swh.web.save_code_now.models import ( + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_PENDING, + SAVE_REQUEST_REJECTED, + SAVE_TASK_FAILED, + SAVE_TASK_NOT_CREATED, + SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_RUNNING, + SAVE_TASK_SCHEDULED, + SAVE_TASK_SUCCEEDED, + SaveOriginRequest, +) +from swh.web.save_code_now.origin_save import get_savable_visit_types + +SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) + +SUBMITTED_SAVE_REQUESTS_METRIC = "swh_web_submitted_save_requests" + +_submitted_save_requests_gauge = Gauge( + name=SUBMITTED_SAVE_REQUESTS_METRIC, + documentation="Number of submitted origin save requests", + labelnames=["status", "visit_type"], + registry=SWH_WEB_METRICS_REGISTRY, +) + + +ACCEPTED_SAVE_REQUESTS_METRIC = "swh_web_accepted_save_requests" + +_accepted_save_requests_gauge = Gauge( + name=ACCEPTED_SAVE_REQUESTS_METRIC, + documentation="Number of accepted origin save requests", + labelnames=["load_task_status", "visit_type"], + registry=SWH_WEB_METRICS_REGISTRY, +) + + +# Metric on the delay of save code now request per status and visit_type. This is the +# time difference between the save code now is requested and the time it got ingested. +ACCEPTED_SAVE_REQUESTS_DELAY_METRIC = "swh_web_save_requests_delay_seconds" + +_accepted_save_requests_delay_gauge = Gauge( + name=ACCEPTED_SAVE_REQUESTS_DELAY_METRIC, + documentation="Save Requests Duration", + labelnames=["load_task_status", "visit_type"], + registry=SWH_WEB_METRICS_REGISTRY, +) + + +def compute_save_requests_metrics() -> None: + """Compute Prometheus metrics related to origin save requests: + + - Number of submitted origin save requests + - Number of accepted origin save requests + - Save Code Now requests delay between request time and actual time of ingestion + + """ + + request_statuses = ( + SAVE_REQUEST_ACCEPTED, + SAVE_REQUEST_REJECTED, + SAVE_REQUEST_PENDING, + ) + + load_task_statuses = ( + SAVE_TASK_NOT_CREATED, + SAVE_TASK_NOT_YET_SCHEDULED, + SAVE_TASK_SCHEDULED, + SAVE_TASK_SUCCEEDED, + SAVE_TASK_FAILED, + SAVE_TASK_RUNNING, + ) + + # for metrics, we want access to all visit types + visit_types = get_savable_visit_types(privileged_user=True) + + labels_set = product(request_statuses, visit_types) + + for labels in labels_set: + _submitted_save_requests_gauge.labels(*labels).set(0) + + labels_set = product(load_task_statuses, visit_types) + + for labels in labels_set: + _accepted_save_requests_gauge.labels(*labels).set(0) + + duration_load_task_statuses = ( + SAVE_TASK_FAILED, + SAVE_TASK_SUCCEEDED, + ) + + for labels in product(duration_load_task_statuses, visit_types): + _accepted_save_requests_delay_gauge.labels(*labels).set(0) + + for sor in SaveOriginRequest.objects.all(): + if sor.status == SAVE_REQUEST_ACCEPTED: + _accepted_save_requests_gauge.labels( + load_task_status=sor.loading_task_status, + visit_type=sor.visit_type, + ).inc() + + _submitted_save_requests_gauge.labels( + status=sor.status, visit_type=sor.visit_type + ).inc() + + if ( + sor.loading_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED) + and sor.visit_date is not None + and sor.request_date is not None + ): + delay = sor.visit_date.timestamp() - sor.request_date.timestamp() + _accepted_save_requests_delay_gauge.labels( + load_task_status=sor.loading_task_status, + visit_type=sor.visit_type, + ).inc(delay) diff --git a/swh/web/metrics/urls.py b/swh/web/metrics/urls.py new file mode 100644 --- /dev/null +++ b/swh/web/metrics/urls.py @@ -0,0 +1,12 @@ +# Copyright (C) 2022 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU Affero General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from django.urls import re_path as url + +from swh.web.metrics.views import prometheus_metrics + +urlpatterns = [ + url(r"^metrics/prometheus/$", prometheus_metrics, name="metrics-prometheus"), +] diff --git a/swh/web/misc/metrics.py b/swh/web/metrics/views.py rename from swh/web/misc/metrics.py rename to swh/web/metrics/views.py --- a/swh/web/misc/metrics.py +++ b/swh/web/metrics/views.py @@ -1,4 +1,4 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2022 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU Affero General Public License version 3, or any later version # See top-level LICENSE file for more information @@ -7,8 +7,10 @@ from django.http import HttpResponse -from swh.web.save_code_now.origin_save import compute_save_requests_metrics -from swh.web.utils import SWH_WEB_METRICS_REGISTRY +from swh.web.metrics.prometheus import ( + SWH_WEB_METRICS_REGISTRY, + compute_save_requests_metrics, +) def prometheus_metrics(request): diff --git a/swh/web/misc/urls.py b/swh/web/misc/urls.py --- a/swh/web/misc/urls.py +++ b/swh/web/misc/urls.py @@ -15,7 +15,6 @@ from django.views.decorators.clickjacking import xframe_options_exempt from swh.web.config import get_config -from swh.web.misc.metrics import prometheus_metrics from swh.web.utils import archive from swh.web.utils.exc import sentry_capture_exception @@ -63,7 +62,6 @@ urlpatterns = [ url(r"^jslicenses/$", _jslicenses, name="jslicenses"), url(r"^stat_counters/$", _stat_counters, name="stat-counters"), - url(r"^metrics/prometheus/$", prometheus_metrics, name="metrics-prometheus"), url(r"^", include("swh.web.misc.fundraising")), url(r"^hiring/banner/$", hiring_banner, name="swh-hiring-banner"), ] @@ -72,6 +70,7 @@ # when running end to end tests through cypress, declare some extra # endpoints to provide input data for some of those tests if get_config()["e2e_tests_mode"]: + from swh.web.tests.views import ( get_content_code_data_all_exts, get_content_code_data_all_filenames, diff --git a/swh/web/save_code_now/origin_save.py b/swh/web/save_code_now/origin_save.py --- a/swh/web/save_code_now/origin_save.py +++ b/swh/web/save_code_now/origin_save.py @@ -5,13 +5,11 @@ from datetime import datetime, timedelta, timezone from functools import lru_cache -from itertools import product import json import logging from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse -from prometheus_client import Gauge import requests from django.core.exceptions import ObjectDoesNotExist, ValidationError @@ -26,7 +24,6 @@ SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED, SAVE_TASK_FAILED, - SAVE_TASK_NOT_CREATED, SAVE_TASK_NOT_YET_SCHEDULED, SAVE_TASK_RUNNING, SAVE_TASK_SCHEDULED, @@ -37,7 +34,7 @@ SaveOriginRequest, SaveUnauthorizedOrigin, ) -from swh.web.utils import SWH_WEB_METRICS_REGISTRY, archive, parse_iso8601_date_to_utc +from swh.web.utils import archive, parse_iso8601_date_to_utc from swh.web.utils.exc import ( BadInputExc, ForbiddenExc, @@ -839,103 +836,3 @@ task_info["message"] = message return task_info - - -SUBMITTED_SAVE_REQUESTS_METRIC = "swh_web_submitted_save_requests" - -_submitted_save_requests_gauge = Gauge( - name=SUBMITTED_SAVE_REQUESTS_METRIC, - documentation="Number of submitted origin save requests", - labelnames=["status", "visit_type"], - registry=SWH_WEB_METRICS_REGISTRY, -) - - -ACCEPTED_SAVE_REQUESTS_METRIC = "swh_web_accepted_save_requests" - -_accepted_save_requests_gauge = Gauge( - name=ACCEPTED_SAVE_REQUESTS_METRIC, - documentation="Number of accepted origin save requests", - labelnames=["load_task_status", "visit_type"], - registry=SWH_WEB_METRICS_REGISTRY, -) - - -# Metric on the delay of save code now request per status and visit_type. This is the -# time difference between the save code now is requested and the time it got ingested. -ACCEPTED_SAVE_REQUESTS_DELAY_METRIC = "swh_web_save_requests_delay_seconds" - -_accepted_save_requests_delay_gauge = Gauge( - name=ACCEPTED_SAVE_REQUESTS_DELAY_METRIC, - documentation="Save Requests Duration", - labelnames=["load_task_status", "visit_type"], - registry=SWH_WEB_METRICS_REGISTRY, -) - - -def compute_save_requests_metrics() -> None: - """Compute Prometheus metrics related to origin save requests: - - - Number of submitted origin save requests - - Number of accepted origin save requests - - Save Code Now requests delay between request time and actual time of ingestion - - """ - - request_statuses = ( - SAVE_REQUEST_ACCEPTED, - SAVE_REQUEST_REJECTED, - SAVE_REQUEST_PENDING, - ) - - load_task_statuses = ( - SAVE_TASK_NOT_CREATED, - SAVE_TASK_NOT_YET_SCHEDULED, - SAVE_TASK_SCHEDULED, - SAVE_TASK_SUCCEEDED, - SAVE_TASK_FAILED, - SAVE_TASK_RUNNING, - ) - - # for metrics, we want access to all visit types - visit_types = get_savable_visit_types(privileged_user=True) - - labels_set = product(request_statuses, visit_types) - - for labels in labels_set: - _submitted_save_requests_gauge.labels(*labels).set(0) - - labels_set = product(load_task_statuses, visit_types) - - for labels in labels_set: - _accepted_save_requests_gauge.labels(*labels).set(0) - - duration_load_task_statuses = ( - SAVE_TASK_FAILED, - SAVE_TASK_SUCCEEDED, - ) - - for labels in product(duration_load_task_statuses, visit_types): - _accepted_save_requests_delay_gauge.labels(*labels).set(0) - - for sor in SaveOriginRequest.objects.all(): - if sor.status == SAVE_REQUEST_ACCEPTED: - _accepted_save_requests_gauge.labels( - load_task_status=sor.loading_task_status, - visit_type=sor.visit_type, - ).inc() - - _submitted_save_requests_gauge.labels( - status=sor.status, visit_type=sor.visit_type - ).inc() - - if ( - sor.loading_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED) - and sor.visit_date is not None - and sor.request_date is not None - ): - delay = sor.visit_date.timestamp() - sor.request_date.timestamp() - _accepted_save_requests_delay_gauge.labels( - load_task_status=sor.loading_task_status, - visit_type=sor.visit_type, - ).inc(delay) diff --git a/swh/web/tests/metrics/__init__.py b/swh/web/tests/metrics/__init__.py new file mode 100644 diff --git a/swh/web/tests/misc/test_metrics.py b/swh/web/tests/metrics/test_metrics.py rename from swh/web/tests/misc/test_metrics.py rename to swh/web/tests/metrics/test_metrics.py --- a/swh/web/tests/misc/test_metrics.py +++ b/swh/web/tests/metrics/test_metrics.py @@ -10,6 +10,12 @@ from prometheus_client.exposition import CONTENT_TYPE_LATEST import pytest +from swh.web.metrics.prometheus import ( + ACCEPTED_SAVE_REQUESTS_DELAY_METRIC, + ACCEPTED_SAVE_REQUESTS_METRIC, + SUBMITTED_SAVE_REQUESTS_METRIC, + get_savable_visit_types, +) from swh.web.save_code_now.models import ( SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_PENDING, @@ -22,12 +28,6 @@ SAVE_TASK_SUCCEEDED, SaveOriginRequest, ) -from swh.web.save_code_now.origin_save import ( - ACCEPTED_SAVE_REQUESTS_DELAY_METRIC, - ACCEPTED_SAVE_REQUESTS_METRIC, - SUBMITTED_SAVE_REQUESTS_METRIC, - get_savable_visit_types, -) from swh.web.tests.django_asserts import assert_contains from swh.web.tests.helpers import check_http_get_response from swh.web.utils import reverse diff --git a/swh/web/tests/mailmap/test_app.py b/swh/web/tests/misc/test_app.py rename from swh/web/tests/mailmap/test_app.py rename to swh/web/tests/misc/test_app.py --- a/swh/web/tests/mailmap/test_app.py +++ b/swh/web/tests/misc/test_app.py @@ -7,26 +7,18 @@ from django.urls import get_resolver -from swh.web.mailmap.urls import urlpatterns -from swh.web.tests.django_asserts import assert_not_contains -from swh.web.tests.helpers import check_html_get_response -from swh.web.utils import reverse +from swh.web.metrics.urls import urlpatterns @pytest.mark.django_db def test_mailmap_deactivate(client, mailmap_admin, django_settings): - """Check mailmap feature is deactivated when the swh.web.mailmap django + """Check metrics feature is deactivated when the swh.web.metrics django application is not in installed apps.""" django_settings.SWH_DJANGO_APPS = [ - app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.mailmap" + app for app in django_settings.SWH_DJANGO_APPS if app != "swh.web.metrics" ] - url = reverse("swh-web-homepage") - client.force_login(mailmap_admin) - resp = check_html_get_response(client, url, status_code=200) - assert_not_contains(resp, "swh-mailmap-admin-item") - - mailmap_view_names = set(urlpattern.name for urlpattern in urlpatterns) + metrics_view_names = set(urlpattern.name for urlpattern in urlpatterns) all_view_names = set(get_resolver().reverse_dict.keys()) - assert mailmap_view_names & all_view_names == set() + assert metrics_view_names & all_view_names == set() diff --git a/swh/web/urls.py b/swh/web/urls.py --- a/swh/web/urls.py +++ b/swh/web/urls.py @@ -4,6 +4,7 @@ # See top-level LICENSE file for more information from importlib.util import find_spec +from typing import List, Union from django_js_reverse.views import urls_js @@ -11,6 +12,7 @@ from django.conf.urls import handler400, handler403, handler404, handler500, include from django.contrib.staticfiles.views import serve from django.shortcuts import render +from django.urls import URLPattern, URLResolver from django.urls import re_path as url from django.views.generic.base import RedirectView @@ -30,7 +32,19 @@ return render(request, "homepage.html", {"visit_types": origin_visit_types()}) -urlpatterns = [ +urlpatterns: List[Union[URLPattern, URLResolver]] = [] + +# Register URLs for each SWH Django application +for app in settings.SWH_DJANGO_APPS: + app_urls = app + ".urls" + try: + app_urls_spec = find_spec(app_urls) + if app_urls_spec is not None: + urlpatterns.append(url(r"^", include(app_urls))) + except ModuleNotFoundError: + assert False, f"Django application {app} not found !" + +urlpatterns += [ url(r"^favicon\.ico/$", favicon_view), url(r"^$", _default_view, name="swh-web-homepage"), url(r"^jsreverse/$", urls_js, name="js_reverse"), @@ -48,16 +62,6 @@ url(r"^", include("swh.web.misc.urls")), ] -# Register URLs for each SWH Django application -for app in settings.SWH_DJANGO_APPS: - app_urls = app + ".urls" - try: - app_urls_spec = find_spec(app_urls) - if app_urls_spec is not None: - urlpatterns.append(url(r"^", include(app_urls))) - except ModuleNotFoundError: - assert False, f"Django application {app} not found !" - # allow to serve assets through django staticfiles # even if settings.DEBUG is False diff --git a/swh/web/utils/__init__.py b/swh/web/utils/__init__.py --- a/swh/web/utils/__init__.py +++ b/swh/web/utils/__init__.py @@ -17,7 +17,6 @@ from docutils.writers.html5_polyglot import HTMLTranslator, Writer from iso8601 import ParseError, parse_date from pkg_resources import get_distribution -from prometheus_client.registry import CollectorRegistry import requests from requests.auth import HTTPBasicAuth @@ -37,8 +36,6 @@ from swh.web.config import SWH_WEB_SERVER_NAME, get_config, search from swh.web.utils.exc import BadInputExc, sentry_capture_exception -SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True) - SWHID_RE = "swh:1:[a-z]{3}:[0-9a-z]{40}"