diff --git a/swh/web/admin/origin_save.py b/swh/web/admin/origin_save.py --- a/swh/web/admin/origin_save.py +++ b/swh/web/admin/origin_save.py @@ -21,6 +21,7 @@ from swh.web.common.origin_save import ( create_save_origin_request, get_save_origin_task_info, + update_pending_save_requests_stats, SAVE_REQUEST_PENDING, SAVE_REQUEST_REJECTED ) @@ -83,6 +84,7 @@ for psr in pending_save_requests: create_save_origin_request(psr.visit_type, psr.origin_url) status_code = 200 + update_pending_save_requests_stats() else: status_code = 400 return HttpResponse(status=status_code) @@ -129,6 +131,7 @@ psr.status = SAVE_REQUEST_REJECTED psr.save() status_code = 200 + update_pending_save_requests_stats() else: status_code = 400 return HttpResponse(status=status_code) @@ -159,6 +162,7 @@ except ObjectDoesNotExist: SaveAuthorizedOrigin.objects.create(url=origin_url) create_save_origin_request(visit_type, origin_url) + update_pending_save_requests_stats() return HttpResponse(status=200) @@ -176,6 +180,7 @@ status=SAVE_REQUEST_PENDING) sor.status = SAVE_REQUEST_REJECTED sor.save() + update_pending_save_requests_stats() return HttpResponse(status=200) diff --git a/swh/web/common/origin_save.py b/swh/web/common/origin_save.py --- a/swh/web/common/origin_save.py +++ b/swh/web/common/origin_save.py @@ -16,6 +16,8 @@ from django.core.validators import URLValidator from django.utils.html import escape +from swh.core.statsd import statsd + from swh.web import config from swh.web.common import service from swh.web.common.exc import BadInputExc, ForbiddenExc, NotFoundExc @@ -34,6 +36,9 @@ logger = logging.getLogger(__name__) +SAVE_REQUESTS_COUNT_METRIC = 'swh_web_save_requests' +PENDING_SAVE_REQUESTS_COUNT_METRIC = 'swh_web_pending_save_requests' + def get_origin_save_authorized_urls(): """ @@ -282,7 +287,7 @@ kwargs['svn_url'] = origin_url sor = None - # get list of previously sumitted save requests + # get list of previously submitted save requests current_sors = \ list(SaveOriginRequest.objects.filter(visit_type=visit_type, origin_url=origin_url)) @@ -349,6 +354,13 @@ origin_url=origin_url, status=save_request_status) + statsd.increment(SAVE_REQUESTS_COUNT_METRIC, + tags={'status': save_request_status, + 'visit_type': visit_type}) + + if save_request_status == SAVE_REQUEST_PENDING: + update_pending_save_requests_stats() + if save_request_status == SAVE_REQUEST_REJECTED: raise ForbiddenExc(('The "save code now" request has been rejected ' 'because the provided origin url is blacklisted.')) @@ -533,3 +545,10 @@ pass return task_run + + +def update_pending_save_requests_stats(): + pending_save_requests = SaveOriginRequest.objects.filter( + status=SAVE_REQUEST_PENDING) + statsd.gauge(PENDING_SAVE_REQUESTS_COUNT_METRIC, + pending_save_requests.count()) diff --git a/swh/web/tests/admin/test_origin_save.py b/swh/web/tests/admin/test_origin_save.py --- a/swh/web/tests/admin/test_origin_save.py +++ b/swh/web/tests/admin/test_origin_save.py @@ -12,7 +12,8 @@ from swh.web.common.models import ( SaveAuthorizedOrigin, SaveUnauthorizedOrigin, SaveOriginRequest ) -from swh.web.common.origin_save import can_save_origin +from swh.web.common.origin_save import ( + can_save_origin, PENDING_SAVE_REQUESTS_COUNT_METRIC) from swh.web.common.models import ( SAVE_REQUEST_PENDING, SAVE_REQUEST_ACCEPTED, SAVE_REQUEST_REJECTED, SAVE_TASK_NOT_YET_SCHEDULED @@ -47,7 +48,7 @@ assert unquote(response.url) == login_url -def test_add_authorized_origin_url(client): +def test_add_authorized_origin_url(client, mocker): authorized_url = 'https://scm.adullact.net/anonscm/' assert can_save_origin(authorized_url) == SAVE_REQUEST_PENDING @@ -58,11 +59,15 @@ assert can_save_origin(authorized_url) == SAVE_REQUEST_PENDING + mock_statsd = mocker.patch('swh.web.common.origin_save.statsd') + client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(authorized_url) == SAVE_REQUEST_ACCEPTED + _check_pending_requests_stats_sent(mock_statsd) + def test_remove_authorized_origin_url(client): assert can_save_origin(_authorized_origin_url) == SAVE_REQUEST_ACCEPTED @@ -80,7 +85,7 @@ assert can_save_origin(_authorized_origin_url) == SAVE_REQUEST_PENDING -def test_add_unauthorized_origin_url(client): +def test_add_unauthorized_origin_url(client, mocker): unauthorized_url = 'https://www.yahoo./' assert can_save_origin(unauthorized_url) == SAVE_REQUEST_PENDING @@ -91,11 +96,15 @@ assert can_save_origin(unauthorized_url) == SAVE_REQUEST_PENDING + mock_statsd = mocker.patch('swh.web.common.origin_save.statsd') + client.login(username=_user_name, password=_user_password) response = client.post(url) assert response.status_code == 200 assert can_save_origin(unauthorized_url) == SAVE_REQUEST_REJECTED + _check_pending_requests_stats_sent(mock_statsd) + def test_remove_unauthorized_origin_url(client): assert can_save_origin(_unauthorized_origin_url) == SAVE_REQUEST_REJECTED @@ -144,12 +153,14 @@ }, 'status': 'next_run_not_scheduled', 'id': 1, - } + } ] mock_scheduler.create_tasks.return_value = tasks_data mock_scheduler.get_tasks.return_value = tasks_data + mock_statsd = mocker.patch('swh.web.common.origin_save.statsd') + client.login(username=_user_name, password=_user_password) response = client.post(accept_request_url) assert response.status_code == 200 @@ -159,6 +170,8 @@ assert response.data[0]['save_request_status'] == SAVE_REQUEST_ACCEPTED assert response.data[0]['save_task_status'] == SAVE_TASK_NOT_YET_SCHEDULED + _check_pending_requests_stats_sent(mock_statsd) + def test_reject_pending_save_request(client, mocker): mock_scheduler = mocker.patch('swh.web.common.origin_save.scheduler') @@ -178,6 +191,8 @@ check_not_login(client, reject_request_url) + mock_statsd = mocker.patch('swh.web.common.origin_save.statsd') + client.login(username=_user_name, password=_user_password) response = client.post(reject_request_url) assert response.status_code == 200 @@ -205,6 +220,8 @@ assert response.status_code == 200 assert response.data[0]['save_request_status'] == SAVE_REQUEST_REJECTED + _check_pending_requests_stats_sent(mock_statsd) + def test_remove_save_request(client): sor = SaveOriginRequest.objects.create(visit_type='git', @@ -221,3 +238,11 @@ response = client.post(remove_request_url) assert response.status_code == 200 assert SaveOriginRequest.objects.count() == 0 + + +def _check_pending_requests_stats_sent(mock_statsd): + pending_save_requests = SaveOriginRequest.objects.filter( + status=SAVE_REQUEST_PENDING) + mock_statsd.gauge.assert_called_once_with( + PENDING_SAVE_REQUESTS_COUNT_METRIC, + pending_save_requests.count()) diff --git a/swh/web/tests/api/views/test_origin_save.py b/swh/web/tests/api/views/test_origin_save.py --- a/swh/web/tests/api/views/test_origin_save.py +++ b/swh/web/tests/api/views/test_origin_save.py @@ -8,6 +8,9 @@ from datetime import datetime, timedelta from django.utils import timezone +from swh.web.common.origin_save import ( + SAVE_REQUESTS_COUNT_METRIC, PENDING_SAVE_REQUESTS_COUNT_METRIC +) from swh.web.common.utils import reverse from swh.web.common.models import ( SaveUnauthorizedOrigin, SaveOriginRequest, @@ -86,6 +89,8 @@ 'id': 1, }] + mock_statsd = mocker.patch('swh.web.common.origin_save.statsd') + url = reverse('api-1-save-origin', url_args={'visit_type': 'git', 'origin_url': origin_url}) @@ -103,6 +108,18 @@ else: assert response.status_code == 403, response.data + mock_statsd.increment.assert_called_once_with( + SAVE_REQUESTS_COUNT_METRIC, + tags={'visit_type': 'git', + 'status': expected_request_status}) + + if expected_request_status == SAVE_REQUEST_PENDING: + pending_save_requests = SaveOriginRequest.objects.filter( + status=SAVE_REQUEST_PENDING) + mock_statsd.gauge.assert_called_once_with( + PENDING_SAVE_REQUESTS_COUNT_METRIC, + pending_save_requests.count()) + def check_save_request_status(api_client, mocker, origin_url, expected_request_status, diff --git a/swh/web/tests/data.py b/swh/web/tests/data.py --- a/swh/web/tests/data.py +++ b/swh/web/tests/data.py @@ -168,19 +168,24 @@ # Tests data initialization def _init_tests_data(): - # Load git repositories from archives - loader = GitLoaderFromArchive(config=_TEST_LOADER_CONFIG) - - # Get reference to the memory storage - storage = loader.storage + # To hold reference to the memory storage + storage = None + # Load git repositories from archives for origin in _TEST_ORIGINS: for i, archive in enumerate(origin['archives']): origin_repo_archive = \ os.path.join(os.path.dirname(__file__), 'resources/repos/%s' % archive) - loader.load(origin['url'], origin_repo_archive, - origin['visit_date'][i]) + loader = GitLoaderFromArchive(origin['url'], + archive_path=origin_repo_archive, + config=_TEST_LOADER_CONFIG, + visit_date=origin['visit_date'][i]) + if storage is None: + storage = loader.storage + else: + loader.storage = storage + loader.load() origin.update(storage.origin_get(origin)) # add an 'id' key if enabled