Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Dict | from typing import Dict | ||||
from unittest.mock import call | from unittest.mock import call | ||||
import psycopg2 | import psycopg2 | ||||
import pytest | import pytest | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, Directory, Release, Revision, Snapshot, Origin | Content, Directory, Release, Revision, Snapshot, Origin, OriginVisit | ||||
) | ) | ||||
from swh.storage import HashCollision, get_storage | from swh.storage import HashCollision, get_storage | ||||
from swh.storage.exc import StorageArgumentException | from swh.storage.exc import StorageArgumentException | ||||
@pytest.fixture | @pytest.fixture | ||||
def swh_storage(): | def swh_storage(): | ||||
▲ Show 20 Lines • Show All 585 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_origin_visit_update( | ||||
ov = next(swh_storage.origin_visit_get(origin_url)) | ov = next(swh_storage.origin_visit_get(origin_url)) | ||||
assert ov['origin'] == origin_url | assert ov['origin'] == origin_url | ||||
assert ov['visit'] == origin_visit.visit | assert ov['visit'] == origin_visit.visit | ||||
assert ov['status'] == 'ongoing' | assert ov['status'] == 'ongoing' | ||||
assert ov['snapshot'] is None | assert ov['snapshot'] is None | ||||
assert ov['metadata'] is None | assert ov['metadata'] is None | ||||
swh_storage.origin_visit_update(origin, origin_visit, status='full') | swh_storage.origin_visit_update(origin_visit, status='full') | ||||
ov = next(swh_storage.origin_visit_get(origin_url)) | ov = next(swh_storage.origin_visit_get(origin_url)) | ||||
assert ov['origin'] == origin_url | assert ov['origin'] == origin_url | ||||
assert ov['visit'] == origin_visit.visit | assert ov['visit'] == origin_visit.visit | ||||
assert ov['status'] == 'full' | assert ov['status'] == 'full' | ||||
assert ov['snapshot'] is None | assert ov['snapshot'] is None | ||||
assert ov['metadata'] is None | assert ov['metadata'] is None | ||||
def test_retrying_proxy_swh_storage_origin_visit_update_retry( | def test_retrying_proxy_swh_storage_origin_visit_update_retry( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
""" | """ | ||||
sample_origin = sample_data['origin'][1] | origin_visit = OriginVisit.from_dict(sample_data['origin_visit'][0]) | ||||
origin_url = sample_origin['url'] | |||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | ||||
mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
# first try goes ko | # first try goes ko | ||||
HashCollision('origin hash collision'), | HashCollision('origin hash collision'), | ||||
# second try goes ko | # second try goes ko | ||||
psycopg2.IntegrityError('origin already inserted'), | psycopg2.IntegrityError('origin already inserted'), | ||||
# ok then! | # ok then! | ||||
{'origin': origin_url, 'visit': 1} | None | ||||
] | ] | ||||
mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
'.origin_visit_update.retry.sleep') | '.origin_visit_update.retry.sleep') | ||||
visit_id = 1 | swh_storage.origin_visit_update(origin_visit, status='full') | ||||
swh_storage.origin_visit_update(origin_url, visit_id, status='full') | |||||
mock_memory.assert_has_calls([ | mock_memory.assert_has_calls([ | ||||
call(origin_url, visit_id, metadata=None, | call(origin_visit, metadata=None, | ||||
snapshot=None, status='full'), | snapshot=None, status='full'), | ||||
call(origin_url, visit_id, metadata=None, | call(origin_visit, metadata=None, | ||||
snapshot=None, status='full'), | snapshot=None, status='full'), | ||||
call(origin_url, visit_id, metadata=None, | call(origin_visit, metadata=None, | ||||
snapshot=None, status='full'), | snapshot=None, status='full'), | ||||
]) | ]) | ||||
assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
def test_retrying_proxy_swh_storage_origin_visit_update_failure( | def test_retrying_proxy_swh_storage_origin_visit_update_failure( | ||||
swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
origin_visit = OriginVisit.from_dict(sample_data['origin_visit'][0]) | |||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | ||||
mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
'Refuse to add origin always!') | 'Refuse to add origin always!') | ||||
origin_url = sample_data['origin'][0]['url'] | |||||
visit_id = 9 | |||||
with pytest.raises(StorageArgumentException, match='Refuse to add'): | with pytest.raises(StorageArgumentException, match='Refuse to add'): | ||||
swh_storage.origin_visit_update(origin_url, visit_id, 'partial') | swh_storage.origin_visit_update(origin_visit, 'partial') | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | ||||
"""Standard directory_add works as before | """Standard directory_add works as before | ||||
""" | """ | ||||
▲ Show 20 Lines • Show All 301 Lines • Show Last 20 Lines |