Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Dict | |||||
from unittest.mock import call | from unittest.mock import call | ||||
import psycopg2 | import psycopg2 | ||||
import pytest | import pytest | ||||
from swh.model.model import ( | from swh.model.model import ( | ||||
Content, | Content, | ||||
Directory, | Directory, | ||||
▲ Show 20 Lines • Show All 477 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_tool_add_failure(swh_storage, sample_data, mocker): | ||||
assert not tool | assert not tool | ||||
with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
swh_storage.tool_add([sample_tool]) | swh_storage.tool_add([sample_tool]) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def to_provider(provider: Dict) -> Dict: | def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data): | ||||
return { | """Standard metadata_authority_add works as before | ||||
"provider_name": provider["name"], | |||||
"provider_url": provider["url"], | |||||
"provider_type": provider["type"], | |||||
"metadata": provider["metadata"], | |||||
} | |||||
def test_retrying_proxy_storage_metadata_provider_add(swh_storage, sample_data): | |||||
"""Standard metadata_provider_add works as before | |||||
""" | """ | ||||
provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
provider_get = to_provider(provider) | |||||
provider = swh_storage.metadata_provider_get_by(provider_get) | assert not swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
assert not provider | |||||
provider_id = swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
assert provider_id | |||||
actual_provider = swh_storage.metadata_provider_get(provider_id) | actual_authority = swh_storage.metadata_authority_get( | ||||
assert actual_provider | authority["type"], authority["url"] | ||||
actual_provider_id = actual_provider.pop("id") | ) | ||||
assert actual_provider_id == provider_id | assert actual_authority == authority | ||||
assert actual_provider == provider_get | |||||
def test_retrying_proxy_storage_metadata_provider_add_with_retry( | def test_retrying_proxy_storage_metadata_authority_add_with_retry( | ||||
swh_storage, sample_data, mocker, fake_hash_collision | swh_storage, sample_data, mocker, fake_hash_collision | ||||
): | ): | ||||
"""Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
""" | """ | ||||
provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
provider_get = to_provider(provider) | |||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
"swh.storage.in_memory.InMemoryStorage.metadata_provider_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
) | ) | ||||
mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
# first try goes ko | # first try goes ko | ||||
fake_hash_collision, | fake_hash_collision, | ||||
# second try goes ko | # second try goes ko | ||||
psycopg2.IntegrityError("provider_id already inserted"), | psycopg2.IntegrityError("foo bar"), | ||||
# ok then! | # ok then! | ||||
"provider_id", | None, | ||||
] | ] | ||||
from swh.storage.retry import RetryingProxyStorage # noqa | |||||
mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
"swh.storage.retry.RetryingProxyStorage" ".metadata_provider_add.retry.sleep" | "swh.storage.retry.RetryingProxyStorage" ".metadata_authority_add.retry.sleep" | ||||
ardumont: can you please merge the string, that's black doing some joke on us? | |||||
) | ) | ||||
provider = swh_storage.metadata_provider_get_by(provider_get) | assert not swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
assert not provider | |||||
provider_id = swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
assert provider_id == "provider_id" | |||||
provider_arg_names = ("provider_name", "provider_type", "provider_url", "metadata") | authority_arg_names = ("type", "url", "metadata") | ||||
provider_args = [provider_get[key] for key in provider_arg_names] | authority_args = [authority[key] for key in authority_arg_names] | ||||
mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
[call(*provider_args), call(*provider_args), call(*provider_args),] | [call(*authority_args), call(*authority_args), call(*authority_args),] | ||||
) | ) | ||||
assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
def test_retrying_proxy_swh_storage_metadata_provider_add_failure( | def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | ||||
swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
): | ): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
"swh.storage.in_memory.InMemoryStorage.metadata_provider_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
) | ) | ||||
mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
"Refuse to add provider_id always!" | "Refuse to add authority_id always!" | ||||
) | ) | ||||
provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
provider_get = to_provider(provider) | |||||
provider_id = swh_storage.metadata_provider_get_by(provider_get) | swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
assert not provider_id | |||||
with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_storage_origin_metadata_add(swh_storage, sample_data): | def test_retrying_proxy_storage_origin_metadata_add(swh_storage, sample_data): | ||||
"""Standard origin_metadata_add works as before | """Standard origin_metadata_add works as before | ||||
""" | """ | ||||
ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
swh_storage.origin_add_one(origin) | swh_storage.metadata_authority_add(**sample_data["authority"][0]) | ||||
provider_get = to_provider(ori_meta["provider"]) | swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0]) | ||||
provider_id = swh_storage.metadata_provider_add(**provider_get) | |||||
origin_metadata = swh_storage.origin_metadata_get_by(origin["url"]) | origin_metadata = swh_storage.origin_metadata_get( | ||||
ori_meta["origin_url"], ori_meta["authority"] | |||||
) | |||||
assert not origin_metadata | assert not origin_metadata | ||||
swh_storage.origin_metadata_add( | swh_storage.origin_metadata_add(**ori_meta) | ||||
origin["url"], | |||||
ori_meta["discovery_date"], | |||||
provider_id, | |||||
ori_meta["tool"], | |||||
ori_meta["metadata"], | |||||
) | |||||
origin_metadata = swh_storage.origin_metadata_get_by(origin["url"]) | origin_metadata = swh_storage.origin_metadata_get( | ||||
ori_meta["origin_url"], ori_meta["authority"] | |||||
) | |||||
assert origin_metadata | assert origin_metadata | ||||
def test_retrying_proxy_storage_origin_metadata_add_with_retry( | def test_retrying_proxy_storage_origin_metadata_add_with_retry( | ||||
swh_storage, sample_data, mocker, fake_hash_collision | swh_storage, sample_data, mocker, fake_hash_collision | ||||
): | ): | ||||
"""Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
""" | """ | ||||
ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
swh_storage.origin_add_one(origin) | swh_storage.metadata_authority_add(**sample_data["authority"][0]) | ||||
provider_get = to_provider(ori_meta["provider"]) | swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0]) | ||||
provider_id = swh_storage.metadata_provider_add(**provider_get) | |||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
"swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | ||||
) | ) | ||||
mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
# first try goes ko | # first try goes ko | ||||
fake_hash_collision, | fake_hash_collision, | ||||
# second try goes ko | # second try goes ko | ||||
psycopg2.IntegrityError("provider_id already inserted"), | psycopg2.IntegrityError("foo bar"), | ||||
# ok then! | # ok then! | ||||
None, | None, | ||||
] | ] | ||||
mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
"swh.storage.retry.RetryingProxyStorage" ".origin_metadata_add.retry.sleep" | "swh.storage.retry.RetryingProxyStorage" ".origin_metadata_add.retry.sleep" | ||||
) | ) | ||||
url = origin["url"] | |||||
ts = ori_meta["discovery_date"] | |||||
tool_id = ori_meta["tool"] | |||||
metadata = ori_meta["metadata"] | |||||
# No exception raised as insertion finally came through | # No exception raised as insertion finally came through | ||||
swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, metadata) | swh_storage.origin_metadata_add(**ori_meta) | ||||
mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
[ # 3 calls, as long as error raised | [ # 3 calls, as long as error raised | ||||
call(url, ts, provider_id, tool_id, metadata), | call( | ||||
call(url, ts, provider_id, tool_id, metadata), | ori_meta["origin_url"], | ||||
call(url, ts, provider_id, tool_id, metadata), | ori_meta["discovery_date"], | ||||
ori_meta["authority"], | |||||
ori_meta["fetcher"], | |||||
ori_meta["format"], | |||||
ori_meta["metadata"], | |||||
), | |||||
call( | |||||
ori_meta["origin_url"], | |||||
ori_meta["discovery_date"], | |||||
ori_meta["authority"], | |||||
ori_meta["fetcher"], | |||||
ori_meta["format"], | |||||
ori_meta["metadata"], | |||||
), | |||||
call( | |||||
ori_meta["origin_url"], | |||||
ori_meta["discovery_date"], | |||||
ori_meta["authority"], | |||||
ori_meta["fetcher"], | |||||
ori_meta["format"], | |||||
ori_meta["metadata"], | |||||
), | |||||
] | ] | ||||
) | ) | ||||
assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | ||||
swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
): | ): | ||||
"""Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
""" | """ | ||||
mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
"swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | ||||
) | ) | ||||
mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | ||||
ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
swh_storage.origin_add_one(origin) | |||||
url = origin["url"] | |||||
ts = ori_meta["discovery_date"] | |||||
provider_id = "provider_id" | |||||
tool_id = ori_meta["tool"] | |||||
metadata = ori_meta["metadata"] | |||||
with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, metadata) | swh_storage.origin_metadata_add(**ori_meta) | ||||
assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | ||||
"""Standard origin_visit_update works as before | """Standard origin_visit_update works as before | ||||
""" | """ | ||||
▲ Show 20 Lines • Show All 389 Lines • Show Last 20 Lines |
can you please merge the string, that's black doing some joke on us?