Changeset View
Changeset View
Standalone View
Standalone View
swh/web/tests/misc/test_metrics.py
| # Copyright (C) 2019 The Software Heritage developers | # Copyright (C) 2019-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from datetime import timedelta | |||||
| from itertools import product | from itertools import product | ||||
| import random | import random | ||||
| from prometheus_client.exposition import CONTENT_TYPE_LATEST | from prometheus_client.exposition import CONTENT_TYPE_LATEST | ||||
| import pytest | import pytest | ||||
| from swh.web.common.models import ( | from swh.web.common.models import ( | ||||
| SAVE_REQUEST_ACCEPTED, | SAVE_REQUEST_ACCEPTED, | ||||
| SAVE_REQUEST_PENDING, | SAVE_REQUEST_PENDING, | ||||
| SAVE_REQUEST_REJECTED, | SAVE_REQUEST_REJECTED, | ||||
| SAVE_TASK_FAILED, | SAVE_TASK_FAILED, | ||||
| SAVE_TASK_NOT_CREATED, | SAVE_TASK_NOT_CREATED, | ||||
| SAVE_TASK_NOT_YET_SCHEDULED, | SAVE_TASK_NOT_YET_SCHEDULED, | ||||
| SAVE_TASK_RUNNING, | SAVE_TASK_RUNNING, | ||||
| SAVE_TASK_SCHEDULED, | SAVE_TASK_SCHEDULED, | ||||
| SAVE_TASK_SUCCEEDED, | SAVE_TASK_SUCCEEDED, | ||||
| SaveOriginRequest, | SaveOriginRequest, | ||||
| ) | ) | ||||
| from swh.web.common.origin_save import ( | from swh.web.common.origin_save import ( | ||||
| ACCEPTED_SAVE_REQUESTS_DELAY_METRIC, | |||||
| ACCEPTED_SAVE_REQUESTS_METRIC, | ACCEPTED_SAVE_REQUESTS_METRIC, | ||||
| SUBMITTED_SAVE_REQUESTS_METRIC, | SUBMITTED_SAVE_REQUESTS_METRIC, | ||||
| get_savable_visit_types, | get_savable_visit_types, | ||||
| ) | ) | ||||
| from swh.web.common.utils import reverse | from swh.web.common.utils import reverse | ||||
| from swh.web.tests.django_asserts import assert_contains | from swh.web.tests.django_asserts import assert_contains | ||||
| from swh.web.tests.utils import check_http_get_response | from swh.web.tests.utils import check_http_get_response | ||||
| Show All 16 Lines | load_task_statuses = ( | ||||
| SAVE_TASK_RUNNING, | SAVE_TASK_RUNNING, | ||||
| ) | ) | ||||
| for _ in range(random.randint(50, 100)): | for _ in range(random.randint(50, 100)): | ||||
| visit_type = random.choice(visit_types) | visit_type = random.choice(visit_types) | ||||
| request_satus = random.choice(request_statuses) | request_satus = random.choice(request_statuses) | ||||
| load_task_status = random.choice(load_task_statuses) | load_task_status = random.choice(load_task_statuses) | ||||
| SaveOriginRequest.objects.create( | sor = SaveOriginRequest.objects.create( | ||||
| origin_url="origin", | origin_url="origin", | ||||
| visit_type=visit_type, | visit_type=visit_type, | ||||
| status=request_satus, | status=request_satus, | ||||
| loading_task_status=load_task_status, | loading_task_status=load_task_status, | ||||
| ) | ) | ||||
| if load_task_status in (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED): | |||||
| delay = random.choice(range(60)) | |||||
| sor.visit_date = sor.request_date + timedelta(seconds=delay) | |||||
| sor.save() | |||||
| # Note that this injects dates in the future for the sake of the test only | |||||
| url = reverse("metrics-prometheus") | url = reverse("metrics-prometheus") | ||||
| resp = check_http_get_response( | resp = check_http_get_response( | ||||
| client, url, status_code=200, content_type=CONTENT_TYPE_LATEST | client, url, status_code=200, content_type=CONTENT_TYPE_LATEST | ||||
| ) | ) | ||||
| accepted_requests = SaveOriginRequest.objects.filter(status=SAVE_REQUEST_ACCEPTED) | accepted_requests = SaveOriginRequest.objects.filter(status=SAVE_REQUEST_ACCEPTED) | ||||
| labels_set = product(visit_types, load_task_statuses) | labels_set = product(visit_types, load_task_statuses) | ||||
| Show All 20 Lines | for labels in labels_set: | ||||
| metric_text = ( | metric_text = ( | ||||
| f"{SUBMITTED_SAVE_REQUESTS_METRIC}{{" | f"{SUBMITTED_SAVE_REQUESTS_METRIC}{{" | ||||
| f'status="{labels[1]}",' | f'status="{labels[1]}",' | ||||
| f'visit_type="{labels[0]}"}} {float(sor_count)}\n' | f'visit_type="{labels[0]}"}} {float(sor_count)}\n' | ||||
| ) | ) | ||||
| assert_contains(resp, metric_text) | assert_contains(resp, metric_text) | ||||
| # delay metrics | |||||
| save_requests = SaveOriginRequest.objects.all() | |||||
| labels_set = product(visit_types, (SAVE_TASK_SUCCEEDED, SAVE_TASK_FAILED,)) | |||||
| for labels in labels_set: | |||||
| sors = save_requests.filter( | |||||
| visit_type=labels[0], | |||||
| loading_task_status=labels[1], | |||||
| visit_date__isnull=False, | |||||
| ) | |||||
ardumont: right, i'll need to amend data to actually pass through here then... | |||||
| delay = 0 | |||||
| for sor in sors: | |||||
| delay += sor.visit_date.timestamp() - sor.request_date.timestamp() | |||||
| metric_delay_text = ( | |||||
| f"{ACCEPTED_SAVE_REQUESTS_DELAY_METRIC}{{" | |||||
| f'load_task_status="{labels[1]}",' | |||||
| f'visit_type="{labels[0]}"}} {float(delay)}\n' | |||||
| ) | |||||
| assert_contains(resp, metric_delay_text) | |||||
right, i'll need to amend data to actually pass through here then...