Differential D8224 Diff 29669 swh/storage/tests/test_retry.py

Changeset View

Standalone View

swh/storage/tests/test_retry.py

# Copyright (C) 2020 The Software Heritage developers		# Copyright (C) 2020 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from unittest.mock import call		from unittest.mock import call

import attr		import attr
import psycopg2		import psycopg2
import pytest		import pytest

		from swh.core.api import TransientRemoteException
from swh.storage.exc import HashCollision, StorageArgumentException		from swh.storage.exc import HashCollision, StorageArgumentException
from swh.storage.utils import now		from swh.storage.utils import now


@pytest.fixture		@pytest.fixture
def monkeypatch_sleep(monkeypatch, swh_storage):		def monkeypatch_sleep(monkeypatch, swh_storage):
"""In test context, we don't want to wait, make test faster"""		"""In test context, we don't want to wait, make test faster"""
from swh.storage.proxies.retry import RetryingProxyStorage		from swh.storage.proxies.retry import RetryingProxyStorage
▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines	mock_memory.side_effect = [
# second try goes ko		# second try goes ko
psycopg2.IntegrityError("content already inserted"),		psycopg2.IntegrityError("content already inserted"),
# ok then!		# ok then!
{"content:add": 1},		{"content:add": 1},
]		]

sample_content = sample_data.content		sample_content = sample_data.content

		sleep = mocker.patch("time.sleep")
content = swh_storage.content_get_data(sample_content.sha1)		content = swh_storage.content_get_data(sample_content.sha1)
assert content is None		assert content is None

s = swh_storage.content_add([sample_content])		s = swh_storage.content_add([sample_content])
assert s == {"content:add": 1}		assert s == {"content:add": 1}

mock_memory.assert_has_calls(		mock_memory.assert_has_calls(
[		[
call([sample_content]),		call([sample_content]),
call([sample_content]),		call([sample_content]),
call([sample_content]),		call([sample_content]),
]		]
)		)

		assert len(sleep.mock_calls) == 2
		(_name, args1, _kwargs) = sleep.mock_calls[0]
		(_name, args2, _kwargs) = sleep.mock_calls[1]
		assert 0 < args1[0] < 1
		assert 0 < args2[0] < 2


		def test_retrying_proxy_storage_content_add_with_retry_of_transient(
		monkeypatch_sleep,
		swh_storage,
		sample_data,
		mocker,
		):
		"""Multiple retries for hash collision and psycopg2 error but finally ok
		after many attempts"""
		mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add")
		mock_memory.side_effect = [
		TransientRemoteException("temporary failure"),
		TransientRemoteException("temporary failure"),
		# ok then!
		{"content:add": 1},
		]

		sample_content = sample_data.content

		content = swh_storage.content_get_data(sample_content.sha1)
		assert content is None

		sleep = mocker.patch("time.sleep")
		s = swh_storage.content_add([sample_content])
		assert s == {"content:add": 1}

		mock_memory.assert_has_calls(
		[
		call([sample_content]),
		call([sample_content]),
		call([sample_content]),
		]
		)

		assert len(sleep.mock_calls) == 2
		(_name, args1, _kwargs) = sleep.mock_calls[0]
		(_name, args2, _kwargs) = sleep.mock_calls[1]
		assert 10 < args1[0] < 11
		assert 10 < args2[0] < 12


def test_retrying_proxy_swh_storage_content_add_failure(		def test_retrying_proxy_swh_storage_content_add_failure(
swh_storage, sample_data, mocker		swh_storage, sample_data, mocker
):		):
"""Unfiltered errors are raising without retry"""		"""Unfiltered errors are raising without retry"""
mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add")		mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add")
mock_memory.side_effect = StorageArgumentException("Refuse to add content always!")		mock_memory.side_effect = StorageArgumentException("Refuse to add content always!")

▲ Show 20 Lines • Show All 118 Lines • Show Last 20 Lines