Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from typing import Dict | from typing import Dict | ||||
| from unittest.mock import call | from unittest.mock import call | ||||
| import psycopg2 | import psycopg2 | ||||
| import pytest | import pytest | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| Content, Directory, Release, Revision, Snapshot, Origin | Content, Directory, Release, Revision, Snapshot, Origin | ||||
| ) | ) | ||||
| from swh.storage import HashCollision, get_storage | from swh.storage import get_storage | ||||
| from swh.storage.exc import StorageArgumentException | from swh.storage.exc import HashCollision, StorageArgumentException | ||||
| @pytest.fixture | |||||
| def fake_hash_collision(sample_data): | |||||
| return HashCollision( | |||||
| 'sha1', "38762cf7f55934b34d179ae6a4c80cadccbb7f0a", []) | |||||
| @pytest.fixture | @pytest.fixture | ||||
| def swh_storage(): | def swh_storage(): | ||||
| storage_config = { | storage_config = { | ||||
| 'cls': 'pipeline', | 'cls': 'pipeline', | ||||
| 'steps': [ | 'steps': [ | ||||
| {'cls': 'validate'}, | {'cls': 'validate'}, | ||||
| Show All 19 Lines | assert s == { | ||||
| 'content:add:bytes': sample_content['length'], | 'content:add:bytes': sample_content['length'], | ||||
| } | } | ||||
| content = next(swh_storage.content_get([sample_content['sha1']])) | content = next(swh_storage.content_get([sample_content['sha1']])) | ||||
| assert content['sha1'] == sample_content['sha1'] | assert content['sha1'] == sample_content['sha1'] | ||||
| def test_retrying_proxy_storage_content_add_with_retry( | def test_retrying_proxy_storage_content_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.content_add') | 'swh.storage.in_memory.InMemoryStorage.content_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('content hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('content already inserted'), | psycopg2.IntegrityError('content already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'content:add': 1} | {'content:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.content_add.retry.sleep') | '.content_add.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | ||||
| } | } | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert len(content_metadata[pk]) == 1 | assert len(content_metadata[pk]) == 1 | ||||
| assert content_metadata[pk][0]['sha1'] == pk | assert content_metadata[pk][0]['sha1'] == pk | ||||
| def test_retrying_proxy_storage_content_add_metadata_with_retry( | def test_retrying_proxy_storage_content_add_metadata_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.content_add_metadata') | 'swh.storage.in_memory.InMemoryStorage.content_add_metadata') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('content_metadata hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('content_metadata already inserted'), | psycopg2.IntegrityError('content_metadata already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'content:add': 1} | {'content:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.content_add_metadata.retry.sleep') | '.content_add_metadata.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_origin_add_one(swh_storage, sample_data): | ||||
| r = swh_storage.origin_add_one(sample_origin) | r = swh_storage.origin_add_one(sample_origin) | ||||
| assert r == sample_origin['url'] | assert r == sample_origin['url'] | ||||
| origin = swh_storage.origin_get(sample_origin) | origin = swh_storage.origin_get(sample_origin) | ||||
| assert origin['url'] == sample_origin['url'] | assert origin['url'] == sample_origin['url'] | ||||
| def test_retrying_proxy_swh_storage_origin_add_one_retry( | def test_retrying_proxy_swh_storage_origin_add_one_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_origin = sample_data['origin'][1] | sample_origin = sample_data['origin'][1] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.origin_add_one') | 'swh.storage.in_memory.InMemoryStorage.origin_add_one') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('origin hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('origin already inserted'), | psycopg2.IntegrityError('origin already inserted'), | ||||
| # ok then! | # ok then! | ||||
| sample_origin['url'] | sample_origin['url'] | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.origin_add_one.retry.sleep') | '.origin_add_one.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 48 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | ||||
| assert isinstance(origin_visit.visit, int) | assert isinstance(origin_visit.visit, int) | ||||
| origin_visit = next(swh_storage.origin_visit_get(origin_url)) | origin_visit = next(swh_storage.origin_visit_get(origin_url)) | ||||
| assert origin_visit['origin'] == origin_url | assert origin_visit['origin'] == origin_url | ||||
| assert isinstance(origin_visit['visit'], int) | assert isinstance(origin_visit['visit'], int) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_retry( | def test_retrying_proxy_swh_storage_origin_visit_add_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_origin = sample_data['origin'][1] | sample_origin = sample_data['origin'][1] | ||||
| origin_url = swh_storage.origin_add_one(sample_origin) | origin_url = swh_storage.origin_add_one(sample_origin) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.origin_visit_add') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('origin hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('origin already inserted'), | psycopg2.IntegrityError('origin already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'origin': origin_url, 'visit': 1} | {'origin': origin_url, 'visit': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.origin_visit_add.retry.sleep') | '.origin_visit_add.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | def test_retrying_proxy_storage_tool_add(swh_storage, sample_data): | ||||
| assert tool == sample_tool | assert tool == sample_tool | ||||
| tool = swh_storage.tool_get(sample_tool) | tool = swh_storage.tool_get(sample_tool) | ||||
| tool.pop('id') | tool.pop('id') | ||||
| assert tool == sample_tool | assert tool == sample_tool | ||||
| def test_retrying_proxy_storage_tool_add_with_retry( | def test_retrying_proxy_storage_tool_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_tool = sample_data['tool'][0] | sample_tool = sample_data['tool'][0] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.tool_add') | 'swh.storage.in_memory.InMemoryStorage.tool_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('tool hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('tool already inserted'), | psycopg2.IntegrityError('tool already inserted'), | ||||
| # ok then! | # ok then! | ||||
| [sample_tool] | [sample_tool] | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.tool_add.retry.sleep') | '.tool_add.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 59 Lines • ▼ Show 20 Lines | def test_retrying_proxy_storage_metadata_provider_add( | ||||
| actual_provider = swh_storage.metadata_provider_get(provider_id) | actual_provider = swh_storage.metadata_provider_get(provider_id) | ||||
| assert actual_provider | assert actual_provider | ||||
| actual_provider_id = actual_provider.pop('id') | actual_provider_id = actual_provider.pop('id') | ||||
| assert actual_provider_id == provider_id | assert actual_provider_id == provider_id | ||||
| assert actual_provider == provider_get | assert actual_provider == provider_get | ||||
| def test_retrying_proxy_storage_metadata_provider_add_with_retry( | def test_retrying_proxy_storage_metadata_provider_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| provider = sample_data['provider'][0] | provider = sample_data['provider'][0] | ||||
| provider_get = to_provider(provider) | provider_get = to_provider(provider) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.metadata_provider_add') | 'swh.storage.in_memory.InMemoryStorage.metadata_provider_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('provider_id hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('provider_id already inserted'), | psycopg2.IntegrityError('provider_id already inserted'), | ||||
| # ok then! | # ok then! | ||||
| 'provider_id', | 'provider_id', | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.metadata_provider_add.retry.sleep') | '.metadata_provider_add.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 56 Lines • ▼ Show 20 Lines | swh_storage.origin_metadata_add( | ||||
| provider_id, ori_meta['tool'], ori_meta['metadata']) | provider_id, ori_meta['tool'], ori_meta['metadata']) | ||||
| origin_metadata = swh_storage.origin_metadata_get_by( | origin_metadata = swh_storage.origin_metadata_get_by( | ||||
| origin['url']) | origin['url']) | ||||
| assert origin_metadata | assert origin_metadata | ||||
| def test_retrying_proxy_storage_origin_metadata_add_with_retry( | def test_retrying_proxy_storage_origin_metadata_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| ori_meta = sample_data['origin_metadata'][0] | ori_meta = sample_data['origin_metadata'][0] | ||||
| origin = ori_meta['origin'] | origin = ori_meta['origin'] | ||||
| swh_storage.origin_add_one(origin) | swh_storage.origin_add_one(origin) | ||||
| provider_get = to_provider(ori_meta['provider']) | provider_get = to_provider(ori_meta['provider']) | ||||
| provider_id = swh_storage.metadata_provider_add(**provider_get) | provider_id = swh_storage.metadata_provider_add(**provider_get) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.origin_metadata_add') | 'swh.storage.in_memory.InMemoryStorage.origin_metadata_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('provider_id hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('provider_id already inserted'), | psycopg2.IntegrityError('provider_id already inserted'), | ||||
| # ok then! | # ok then! | ||||
| None | None | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| ▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_origin_visit_update( | ||||
| assert ov['origin'] == origin_url | assert ov['origin'] == origin_url | ||||
| assert ov['visit'] == origin_visit.visit | assert ov['visit'] == origin_visit.visit | ||||
| assert ov['status'] == 'full' | assert ov['status'] == 'full' | ||||
| assert ov['snapshot'] is None | assert ov['snapshot'] is None | ||||
| assert ov['metadata'] is None | assert ov['metadata'] is None | ||||
| def test_retrying_proxy_swh_storage_origin_visit_update_retry( | def test_retrying_proxy_swh_storage_origin_visit_update_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_origin = sample_data['origin'][1] | sample_origin = sample_data['origin'][1] | ||||
| origin_url = sample_origin['url'] | origin_url = sample_origin['url'] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | 'swh.storage.in_memory.InMemoryStorage.origin_visit_update') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('origin hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('origin already inserted'), | psycopg2.IntegrityError('origin already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'origin': origin_url, 'visit': 1} | {'origin': origin_url, 'visit': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| ▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Lines | assert s == { | ||||
| 'directory:add': 1, | 'directory:add': 1, | ||||
| } | } | ||||
| directory_id = swh_storage.directory_get_random() # only 1 | directory_id = swh_storage.directory_get_random() # only 1 | ||||
| assert directory_id == sample_dir['id'] | assert directory_id == sample_dir['id'] | ||||
| def test_retrying_proxy_storage_directory_add_with_retry( | def test_retrying_proxy_storage_directory_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.directory_add') | 'swh.storage.in_memory.InMemoryStorage.directory_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('directory hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('directory already inserted'), | psycopg2.IntegrityError('directory already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'directory:add': 1} | {'directory:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| '.directory_add.retry.sleep') | '.directory_add.retry.sleep') | ||||
| ▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | assert s == { | ||||
| 'revision:add': 1, | 'revision:add': 1, | ||||
| } | } | ||||
| revision = next(swh_storage.revision_get([sample_rev['id']])) | revision = next(swh_storage.revision_get([sample_rev['id']])) | ||||
| assert revision['id'] == sample_rev['id'] | assert revision['id'] == sample_rev['id'] | ||||
| def test_retrying_proxy_storage_revision_add_with_retry( | def test_retrying_proxy_storage_revision_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.revision_add') | 'swh.storage.in_memory.InMemoryStorage.revision_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('revision hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('revision already inserted'), | psycopg2.IntegrityError('revision already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'revision:add': 1} | {'revision:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| ▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | assert s == { | ||||
| 'release:add': 1, | 'release:add': 1, | ||||
| } | } | ||||
| release = next(swh_storage.release_get([sample_rel['id']])) | release = next(swh_storage.release_get([sample_rel['id']])) | ||||
| assert release['id'] == sample_rel['id'] | assert release['id'] == sample_rel['id'] | ||||
| def test_retrying_proxy_storage_release_add_with_retry( | def test_retrying_proxy_storage_release_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.release_add') | 'swh.storage.in_memory.InMemoryStorage.release_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('release hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('release already inserted'), | psycopg2.IntegrityError('release already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'release:add': 1} | {'release:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| ▲ Show 20 Lines • Show All 52 Lines • ▼ Show 20 Lines | assert s == { | ||||
| 'snapshot:add': 1, | 'snapshot:add': 1, | ||||
| } | } | ||||
| snapshot = swh_storage.snapshot_get(sample_snap['id']) | snapshot = swh_storage.snapshot_get(sample_snap['id']) | ||||
| assert snapshot['id'] == sample_snap['id'] | assert snapshot['id'] == sample_snap['id'] | ||||
| def test_retrying_proxy_storage_snapshot_add_with_retry( | def test_retrying_proxy_storage_snapshot_add_with_retry( | ||||
| swh_storage, sample_data, mocker): | swh_storage, sample_data, mocker, fake_hash_collision): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| 'swh.storage.in_memory.InMemoryStorage.snapshot_add') | 'swh.storage.in_memory.InMemoryStorage.snapshot_add') | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| HashCollision('snapshot hash collision'), | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError('snapshot already inserted'), | psycopg2.IntegrityError('snapshot already inserted'), | ||||
| # ok then! | # ok then! | ||||
| {'snapshot:add': 1} | {'snapshot:add': 1} | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| 'swh.storage.retry.RetryingProxyStorage' | 'swh.storage.retry.RetryingProxyStorage' | ||||
| Show All 39 Lines | |||||