Differential D8890 Diff 32078 swh/web/metrics/tests/test_metrics.py

Changeset View

Standalone View

swh/web/metrics/tests/test_metrics.py

# Copyright (C) 2019-2021 The Software Heritage developers		# Copyright (C) 2019-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from datetime import timedelta		from datetime import timedelta
from itertools import product		from itertools import product
import random		import random

from prometheus_client.exposition import CONTENT_TYPE_LATEST		from prometheus_client.exposition import CONTENT_TYPE_LATEST
import pytest		import pytest

from swh.web.metrics.prometheus import (		from swh.web.metrics.prometheus import (
ACCEPTED_SAVE_REQUESTS_DELAY_METRIC,		ACCEPTED_SAVE_REQUESTS_DELAY_METRIC,
ACCEPTED_SAVE_REQUESTS_METRIC,		ACCEPTED_SAVE_REQUESTS_METRIC,
		SUBMITTED_SAVE_REQUESTS_FROM_WEBHOOKS_METRIC,
SUBMITTED_SAVE_REQUESTS_METRIC,		SUBMITTED_SAVE_REQUESTS_METRIC,
get_savable_visit_types,		get_savable_visit_types,
)		)
from swh.web.save_code_now.models import (		from swh.web.save_code_now.models import (
SAVE_REQUEST_ACCEPTED,		SAVE_REQUEST_ACCEPTED,
SAVE_REQUEST_PENDING,		SAVE_REQUEST_PENDING,
SAVE_REQUEST_REJECTED,		SAVE_REQUEST_REJECTED,
SAVE_TASK_FAILED,		SAVE_TASK_FAILED,
SAVE_TASK_NOT_CREATED,		SAVE_TASK_NOT_CREATED,
SAVE_TASK_NOT_YET_SCHEDULED,		SAVE_TASK_NOT_YET_SCHEDULED,
SAVE_TASK_RUNNING,		SAVE_TASK_RUNNING,
SAVE_TASK_SCHEDULED,		SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,		SAVE_TASK_SUCCEEDED,
SaveOriginRequest,		SaveOriginRequest,
)		)
		from swh.web.save_origin_webhooks.generic_receiver import SUPPORTED_FORGE_TYPES
from swh.web.tests.django_asserts import assert_contains		from swh.web.tests.django_asserts import assert_contains
from swh.web.tests.helpers import check_http_get_response		from swh.web.tests.helpers import check_http_get_response
from swh.web.utils import reverse		from swh.web.utils import reverse


@pytest.mark.django_db		@pytest.mark.django_db
def test_origin_save_metrics(client, swh_scheduler):		def test_origin_save_metrics(client, swh_scheduler):
visit_types = get_savable_visit_types()		visit_types = get_savable_visit_types()
request_statuses = (		request_statuses = (
SAVE_REQUEST_ACCEPTED,		SAVE_REQUEST_ACCEPTED,
SAVE_REQUEST_REJECTED,		SAVE_REQUEST_REJECTED,
SAVE_REQUEST_PENDING,		SAVE_REQUEST_PENDING,
)		)

load_task_statuses = (		load_task_statuses = (
SAVE_TASK_NOT_CREATED,		SAVE_TASK_NOT_CREATED,
SAVE_TASK_NOT_YET_SCHEDULED,		SAVE_TASK_NOT_YET_SCHEDULED,
SAVE_TASK_SCHEDULED,		SAVE_TASK_SCHEDULED,
SAVE_TASK_SUCCEEDED,		SAVE_TASK_SUCCEEDED,
SAVE_TASK_FAILED,		SAVE_TASK_FAILED,
SAVE_TASK_RUNNING,		SAVE_TASK_RUNNING,
)		)

for _ in range(random.randint(50, 100)):		supported_forge_types = list(SUPPORTED_FORGE_TYPES)

		for i in range(random.randint(50, 100)):
visit_type = random.choice(visit_types)		visit_type = random.choice(visit_types)
request_satus = random.choice(request_statuses)		request_satus = random.choice(request_statuses)
load_task_status = random.choice(load_task_statuses)		load_task_status = random.choice(load_task_statuses)
		from_webhook = i % 2 == 0

sor = SaveOriginRequest.objects.create(		sor = SaveOriginRequest.objects.create(
origin_url="origin",		origin_url="origin",
visit_type=visit_type,		visit_type=visit_type,
status=request_satus,		status=request_satus,
loading_task_status=load_task_status,		loading_task_status=load_task_status,
		from_webhook=from_webhook,
		webhook_origin=random.choice(supported_forge_types)
		if from_webhook
		else None,
)		)

if load_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED):		if load_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED):
delay = random.choice(range(60))		delay = random.choice(range(60))
sor.visit_date = sor.request_date + timedelta(seconds=delay)		sor.visit_date = sor.request_date + timedelta(seconds=delay)
sor.save()		sor.save()
# Note that this injects dates in the future for the sake of the test only		# Note that this injects dates in the future for the sake of the test only

Show All 29 Lines	for labels in labels_set:
metric_text = (		metric_text = (
f"{SUBMITTED_SAVE_REQUESTS_METRIC}{{"		f"{SUBMITTED_SAVE_REQUESTS_METRIC}{{"
f'status="{labels[1]}",'		f'status="{labels[1]}",'
f'visit_type="{labels[0]}"}} {float(sor_count)}\n'		f'visit_type="{labels[0]}"}} {float(sor_count)}\n'
)		)

assert_contains(resp, metric_text)		assert_contains(resp, metric_text)

		labels_set = product(request_statuses, SUPPORTED_FORGE_TYPES)
		for labels in labels_set:
		sor_count = SaveOriginRequest.objects.filter(
		status=labels[0], webhook_origin=labels[1]
		).count()

		metric_text = (
		f"{SUBMITTED_SAVE_REQUESTS_FROM_WEBHOOKS_METRIC}{{"
		f'status="{labels[0]}",'
		f'webhook_origin="{labels[1]}"}} {float(sor_count)}\n'
		)

		assert_contains(resp, metric_text)

# delay metrics		# delay metrics

save_requests = SaveOriginRequest.objects.all()		save_requests = SaveOriginRequest.objects.all()

labels_set = product(		labels_set = product(
visit_types,		visit_types,
(		(
SAVE_TASK_SUCCEEDED,		SAVE_TASK_SUCCEEDED,
Show All 21 Lines