Differential D8519 Diff 30744 swh/lister/tests/test_utils.py

Changeset View

Standalone View

swh/lister/tests/test_utils.py

# Copyright (C) 2018-2021 the Software Heritage developers		# Copyright (C) 2018-2022 the Software Heritage developers
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import pytest		import pytest
import requests		import requests
from requests.status_codes import codes		from requests.status_codes import codes
from tenacity.wait import wait_fixed		from tenacity.wait import wait_fixed

from swh.lister.utils import (		from swh.lister.utils import MAX_NUMBER_ATTEMPTS, WAIT_EXP_BASE, http_retry, split_range
MAX_NUMBER_ATTEMPTS,
WAIT_EXP_BASE,
split_range,
throttling_retry,
)


@pytest.mark.parametrize(		@pytest.mark.parametrize(
"total_pages,nb_pages,expected_ranges",		"total_pages,nb_pages,expected_ranges",
[		[
(14, 5, [(0, 4), (5, 9), (10, 14)]),		(14, 5, [(0, 4), (5, 9), (10, 14)]),
(19, 10, [(0, 9), (10, 19)]),		(19, 10, [(0, 9), (10, 19)]),
(20, 3, [(0, 2), (3, 5), (6, 8), (9, 11), (12, 14), (15, 17), (18, 20)]),		(20, 3, [(0, 2), (3, 5), (6, 8), (9, 11), (12, 14), (15, 17), (18, 20)]),
Show All 22 Lines	def test_split_range_errors(total_pages, nb_pages):
for total_pages, nb_pages in [(None, 1), (100, None)]:		for total_pages, nb_pages in [(None, 1), (100, None)]:
with pytest.raises(TypeError):		with pytest.raises(TypeError):
next(split_range(total_pages, nb_pages))		next(split_range(total_pages, nb_pages))


TEST_URL = "https://example.og/api/repositories"		TEST_URL = "https://example.og/api/repositories"


@throttling_retry()		@http_retry()
def make_request():		def make_request():
response = requests.get(TEST_URL)		response = requests.get(TEST_URL)
response.raise_for_status()		response.raise_for_status()
return response		return response


def assert_sleep_calls(mocker, mock_sleep, sleep_params):		def assert_sleep_calls(mocker, mock_sleep, sleep_params):
mock_sleep.assert_has_calls([mocker.call(param) for param in sleep_params])		mock_sleep.assert_has_calls([mocker.call(param) for param in sleep_params])


def test_throttling_retry(requests_mock, mocker):		@pytest.mark.parametrize(
		"status_code",
		[
		codes.too_many_requests,
		codes.internal_server_error,
		codes.bad_gateway,
		codes.service_unavailable,
		],
		)
		def test_http_retry(requests_mock, mocker, status_code):
data = {"result": {}}		data = {"result": {}}
requests_mock.get(		requests_mock.get(
TEST_URL,		TEST_URL,
[		[
{"status_code": codes.too_many_requests},		{"status_code": status_code},
{"status_code": codes.too_many_requests},		{"status_code": status_code},
{"status_code": codes.ok, "json": data},		{"status_code": codes.ok, "json": data},
],		],
)		)

mock_sleep = mocker.patch.object(make_request.retry, "sleep")		mock_sleep = mocker.patch.object(make_request.retry, "sleep")

response = make_request()		response = make_request()

assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE])		assert_sleep_calls(mocker, mock_sleep, [1, WAIT_EXP_BASE])

assert response.json() == data		assert response.json() == data


def test_throttling_retry_max_attemps(requests_mock, mocker):		def test_http_retry_max_attemps(requests_mock, mocker):
requests_mock.get(		requests_mock.get(
TEST_URL,		TEST_URL,
[{"status_code": codes.too_many_requests}] * (MAX_NUMBER_ATTEMPTS),		[{"status_code": codes.too_many_requests}] * (MAX_NUMBER_ATTEMPTS),
)		)

mock_sleep = mocker.patch.object(make_request.retry, "sleep")		mock_sleep = mocker.patch.object(make_request.retry, "sleep")

with pytest.raises(requests.exceptions.HTTPError) as e:		with pytest.raises(requests.exceptions.HTTPError) as e:
make_request()		make_request()

assert e.value.response.status_code == codes.too_many_requests		assert e.value.response.status_code == codes.too_many_requests

assert_sleep_calls(		assert_sleep_calls(
mocker,		mocker,
mock_sleep,		mock_sleep,
[float(WAIT_EXP_BASE**i) for i in range(MAX_NUMBER_ATTEMPTS - 1)],		[float(WAIT_EXP_BASE**i) for i in range(MAX_NUMBER_ATTEMPTS - 1)],
)		)


@throttling_retry(wait=wait_fixed(WAIT_EXP_BASE))		@http_retry(wait=wait_fixed(WAIT_EXP_BASE))
def make_request_wait_fixed():		def make_request_wait_fixed():
response = requests.get(TEST_URL)		response = requests.get(TEST_URL)
response.raise_for_status()		response.raise_for_status()
return response		return response


def test_throttling_retry_wait_fixed(requests_mock, mocker):		def test_http_retry_wait_fixed(requests_mock, mocker):
requests_mock.get(		requests_mock.get(
TEST_URL,		TEST_URL,
[		[
{"status_code": codes.too_many_requests},		{"status_code": codes.too_many_requests},
{"status_code": codes.too_many_requests},		{"status_code": codes.too_many_requests},
{"status_code": codes.ok},		{"status_code": codes.ok},
],		],
)		)

mock_sleep = mocker.patch.object(make_request_wait_fixed.retry, "sleep")		mock_sleep = mocker.patch.object(make_request_wait_fixed.retry, "sleep")

make_request_wait_fixed()		make_request_wait_fixed()

assert_sleep_calls(mocker, mock_sleep, [WAIT_EXP_BASE] * 2)		assert_sleep_calls(mocker, mock_sleep, [WAIT_EXP_BASE] * 2)