Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import psycopg2 | import psycopg2 | ||||
import pytest | import pytest | ||||
from typing import Dict | from typing import Dict | ||||
from unittest.mock import call | from unittest.mock import call | ||||
from swh.storage import HashCollision | from swh.storage import HashCollision | ||||
from swh.storage.exc import StorageArgumentException | |||||
from swh.storage.retry import RetryingProxyStorage | from swh.storage.retry import RetryingProxyStorage | ||||
@pytest.fixture | @pytest.fixture | ||||
def swh_storage(): | def swh_storage(): | ||||
return RetryingProxyStorage(storage={'cls': 'memory'}) | return RetryingProxyStorage(storage={'cls': 'memory'}) | ||||
▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_content_add_failure( | def test_retrying_proxy_swh_storage_content_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.content_add') | 'swh.storage.in_memory.InMemoryStorage.content_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add content always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add content always!') | |||||
sample_content = sample_data['content'][0] | sample_content = sample_data['content'][0] | ||||
content = next(swh_storage.content_get([sample_content['sha1']])) | content = next(swh_storage.content_get([sample_content['sha1']])) | ||||
assert not content | assert not content | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.content_add([sample_content]) | swh_storage.content_add([sample_content]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | ||||
"""Standard content_add_metadata works as before | """Standard content_add_metadata works as before | ||||
▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_content_add_metadata_failure( | def test_retrying_proxy_swh_storage_content_add_metadata_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.content_add_metadata') | 'swh.storage.in_memory.InMemoryStorage.content_add_metadata') | ||||
mock_memory.side_effect = ValueError('Refuse to add content_metadata!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add content_metadata!') | |||||
sample_content = sample_data['content_metadata'][0] | sample_content = sample_data['content_metadata'][0] | ||||
pk = sample_content['sha1'] | pk = sample_content['sha1'] | ||||
content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
assert not content_metadata[pk] | assert not content_metadata[pk] | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.content_add_metadata([sample_content]) | swh_storage.content_add_metadata([sample_content]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_swh_storage_origin_add_one(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_add_one(swh_storage, sample_data): | ||||
"""Standard origin_add_one works as before | """Standard origin_add_one works as before | ||||
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_origin_add_one_failure( | def test_retrying_proxy_swh_storage_origin_add_one_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_add_one') | 'swh.storage.in_memory.InMemoryStorage.origin_add_one') | ||||
mock_memory.side_effect = ValueError('Refuse to add origin always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add origin always!') | |||||
sample_origin = sample_data['origin'][0] | sample_origin = sample_data['origin'][0] | ||||
origin = swh_storage.origin_get(sample_origin) | origin = swh_storage.origin_get(sample_origin) | ||||
assert not origin | assert not origin | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.origin_add_one([sample_origin]) | swh_storage.origin_add_one([sample_origin]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | ||||
"""Standard origin_visit_add works as before | """Standard origin_visit_add works as before | ||||
▲ Show 20 Lines • Show All 50 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_origin_visit_add_failure( | def test_retrying_proxy_swh_storage_origin_visit_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_visit_add') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add origin always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add origin always!') | |||||
origin_url = sample_data['origin'][0]['url'] | origin_url = sample_data['origin'][0]['url'] | ||||
origin = list(swh_storage.origin_visit_get(origin_url)) | origin = list(swh_storage.origin_visit_get(origin_url)) | ||||
assert not origin | assert not origin | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.origin_visit_add(origin_url, '2020-01-31', 'svn') | swh_storage.origin_visit_add(origin_url, '2020-01-31', 'svn') | ||||
assert mock_memory.has_calls([ | assert mock_memory.has_calls([ | ||||
call(origin_url, '2020-01-31', 'svn'), | call(origin_url, '2020-01-31', 'svn'), | ||||
]) | ]) | ||||
def test_retrying_proxy_storage_tool_add(swh_storage, sample_data): | def test_retrying_proxy_storage_tool_add(swh_storage, sample_data): | ||||
▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_tool_add_failure( | def test_retrying_proxy_swh_storage_tool_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.tool_add') | 'swh.storage.in_memory.InMemoryStorage.tool_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add tool always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add tool always!') | |||||
sample_tool = sample_data['tool'][0] | sample_tool = sample_data['tool'][0] | ||||
tool = swh_storage.tool_get(sample_tool) | tool = swh_storage.tool_get(sample_tool) | ||||
assert not tool | assert not tool | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.tool_add([sample_tool]) | swh_storage.tool_add([sample_tool]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def to_provider(provider: Dict) -> Dict: | def to_provider(provider: Dict) -> Dict: | ||||
return { | return { | ||||
'provider_name': provider['name'], | 'provider_name': provider['name'], | ||||
▲ Show 20 Lines • Show All 58 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_metadata_provider_add_failure( | def test_retrying_proxy_swh_storage_metadata_provider_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.metadata_provider_add') | 'swh.storage.in_memory.InMemoryStorage.metadata_provider_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add provider_id always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add provider_id always!') | |||||
provider = sample_data['provider'][0] | provider = sample_data['provider'][0] | ||||
provider_get = to_provider(provider) | provider_get = to_provider(provider) | ||||
provider_id = swh_storage.metadata_provider_get_by(provider_get) | provider_id = swh_storage.metadata_provider_get_by(provider_get) | ||||
assert not provider_id | assert not provider_id | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_provider_add(**provider_get) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_origin_metadata_add( | def test_retrying_proxy_storage_origin_metadata_add( | ||||
swh_storage, sample_data): | swh_storage, sample_data): | ||||
"""Standard origin_metadata_add works as before | """Standard origin_metadata_add works as before | ||||
▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_metadata_add') | 'swh.storage.in_memory.InMemoryStorage.origin_metadata_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add always!') | |||||
ori_meta = sample_data['origin_metadata'][0] | ori_meta = sample_data['origin_metadata'][0] | ||||
origin = ori_meta['origin'] | origin = ori_meta['origin'] | ||||
swh_storage.origin_add_one(origin) | swh_storage.origin_add_one(origin) | ||||
url = origin['url'] | url = origin['url'] | ||||
ts = ori_meta['discovery_date'] | ts = ori_meta['discovery_date'] | ||||
provider_id = 'provider_id' | provider_id = 'provider_id' | ||||
tool_id = ori_meta['tool'] | tool_id = ori_meta['tool'] | ||||
metadata = ori_meta['metadata'] | metadata = ori_meta['metadata'] | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, | swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, | ||||
metadata) | metadata) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_swh_storage_origin_visit_update( | def test_retrying_proxy_swh_storage_origin_visit_update( | ||||
swh_storage, sample_data): | swh_storage, sample_data): | ||||
▲ Show 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_origin_visit_update_failure( | def test_retrying_proxy_swh_storage_origin_visit_update_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | ||||
mock_memory.side_effect = ValueError('Refuse to add origin always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add origin always!') | |||||
origin_url = sample_data['origin'][0]['url'] | origin_url = sample_data['origin'][0]['url'] | ||||
visit_id = 9 | visit_id = 9 | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.origin_visit_update(origin_url, visit_id, 'partial') | swh_storage.origin_visit_update(origin_url, visit_id, 'partial') | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | ||||
"""Standard directory_add works as before | """Standard directory_add works as before | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_directory_add_failure( | def test_retrying_proxy_swh_storage_directory_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.directory_add') | 'swh.storage.in_memory.InMemoryStorage.directory_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add directory always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add directory always!') | |||||
sample_dir = sample_data['directory'][0] | sample_dir = sample_data['directory'][0] | ||||
directory_id = swh_storage.directory_get_random() # no directory | directory_id = swh_storage.directory_get_random() # no directory | ||||
assert not directory_id | assert not directory_id | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.directory_add([sample_dir]) | swh_storage.directory_add([sample_dir]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_revision_add(swh_storage, sample_data): | def test_retrying_proxy_storage_revision_add(swh_storage, sample_data): | ||||
"""Standard revision_add works as before | """Standard revision_add works as before | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_revision_add_failure( | def test_retrying_proxy_swh_storage_revision_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.revision_add') | 'swh.storage.in_memory.InMemoryStorage.revision_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add revision always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add revision always!') | |||||
sample_rev = sample_data['revision'][0] | sample_rev = sample_data['revision'][0] | ||||
revision = next(swh_storage.revision_get([sample_rev['id']])) | revision = next(swh_storage.revision_get([sample_rev['id']])) | ||||
assert not revision | assert not revision | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.revision_add([sample_rev]) | swh_storage.revision_add([sample_rev]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_release_add(swh_storage, sample_data): | def test_retrying_proxy_storage_release_add(swh_storage, sample_data): | ||||
"""Standard release_add works as before | """Standard release_add works as before | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_release_add_failure( | def test_retrying_proxy_swh_storage_release_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.release_add') | 'swh.storage.in_memory.InMemoryStorage.release_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add release always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add release always!') | |||||
sample_rel = sample_data['release'][0] | sample_rel = sample_data['release'][0] | ||||
release = next(swh_storage.release_get([sample_rel['id']])) | release = next(swh_storage.release_get([sample_rel['id']])) | ||||
assert not release | assert not release | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.release_add([sample_rel]) | swh_storage.release_add([sample_rel]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data): | def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data): | ||||
"""Standard snapshot_add works as before | """Standard snapshot_add works as before | ||||
▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | |||||
def test_retrying_proxy_swh_storage_snapshot_add_failure( | def test_retrying_proxy_swh_storage_snapshot_add_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.snapshot_add') | 'swh.storage.in_memory.InMemoryStorage.snapshot_add') | ||||
mock_memory.side_effect = ValueError('Refuse to add snapshot always!') | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add snapshot always!') | |||||
sample_snap = sample_data['snapshot'][0] | sample_snap = sample_data['snapshot'][0] | ||||
snapshot = swh_storage.snapshot_get(sample_snap['id']) | snapshot = swh_storage.snapshot_get(sample_snap['id']) | ||||
assert not snapshot | assert not snapshot | ||||
with pytest.raises(ValueError, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.snapshot_add([sample_snap]) | swh_storage.snapshot_add([sample_snap]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 |