Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from typing import Dict | |||||
| from unittest.mock import call | from unittest.mock import call | ||||
| import psycopg2 | import psycopg2 | ||||
| import pytest | import pytest | ||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| Content, | Content, | ||||
| Directory, | Directory, | ||||
| ▲ Show 20 Lines • Show All 407 Lines • ▼ Show 20 Lines | ): | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.origin_visit_add(origin_url, date_visit1, "svn") | swh_storage.origin_visit_add(origin_url, date_visit1, "svn") | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call(origin_url, date_visit1, "svn"),] | [call(origin_url, date_visit1, "svn"),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_storage_tool_add(swh_storage, sample_data): | def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | ||||
| """Standard tool_add works as before | """Standard metadata_fetcher_add works as before | ||||
| """ | """ | ||||
| sample_tool = sample_data["tool"][0] | fetcher = sample_data["fetcher"][0] | ||||
| tool = swh_storage.tool_get(sample_tool) | metadata_fetcher = swh_storage.metadata_fetcher_get( | ||||
| assert not tool | fetcher["name"], fetcher["version"] | ||||
| ) | |||||
| assert not metadata_fetcher | |||||
| tools = swh_storage.tool_add([sample_tool]) | swh_storage.metadata_fetcher_add(**fetcher) | ||||
| assert tools | |||||
| tool = tools[0] | |||||
| tool.pop("id") | |||||
| assert tool == sample_tool | |||||
| tool = swh_storage.tool_get(sample_tool) | actual_fetcher = swh_storage.metadata_fetcher_get( | ||||
| tool.pop("id") | fetcher["name"], fetcher["version"] | ||||
| assert tool == sample_tool | ) | ||||
| assert actual_fetcher == fetcher | |||||
| def test_retrying_proxy_storage_tool_add_with_retry( | def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | ||||
| swh_storage, sample_data, mocker, fake_hash_collision | swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_tool = sample_data["tool"][0] | fetcher = sample_data["fetcher"][0] | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.tool_add") | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | |||||
| ) | |||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("tool already inserted"), | psycopg2.IntegrityError("metadata_fetcher already inserted"), | ||||
| # ok then! | # ok then! | ||||
| [sample_tool], | [fetcher], | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| "swh.storage.retry.RetryingProxyStorage" ".tool_add.retry.sleep" | "swh.storage.retry.RetryingProxyStorage" ".metadata_fetcher_add.retry.sleep" | ||||
| ) | ) | ||||
| tool = swh_storage.tool_get(sample_tool) | actual_fetcher = swh_storage.metadata_fetcher_get( | ||||
| assert not tool | fetcher["name"], fetcher["version"] | ||||
| ) | |||||
| assert not actual_fetcher | |||||
| tools = swh_storage.tool_add([sample_tool]) | swh_storage.metadata_fetcher_add(**fetcher) | ||||
| assert tools == [sample_tool] | |||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_tool]), call([sample_tool]), call([sample_tool]),] | [ | ||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| ] | |||||
| ) | ) | ||||
| assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
| def test_retrying_proxy_swh_storage_tool_add_failure(swh_storage, sample_data, mocker): | def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure( | ||||
| swh_storage, sample_data, mocker | |||||
| ): | |||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.tool_add") | mock_memory = mocker.patch( | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add tool always!") | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | |||||
| mock_memory.side_effect = StorageArgumentException( | |||||
| "Refuse to add metadata_fetcher always!" | |||||
| ) | |||||
| sample_tool = sample_data["tool"][0] | fetcher = sample_data["fetcher"][0] | ||||
| tool = swh_storage.tool_get(sample_tool) | actual_fetcher = swh_storage.metadata_fetcher_get( | ||||
| assert not tool | fetcher["name"], fetcher["version"] | ||||
| ) | |||||
| assert not actual_fetcher | |||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.tool_add([sample_tool]) | swh_storage.metadata_fetcher_add(**fetcher) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def to_provider(provider: Dict) -> Dict: | def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data): | ||||
| return { | """Standard metadata_authority_add works as before | ||||
| "provider_name": provider["name"], | |||||
| "provider_url": provider["url"], | |||||
| "provider_type": provider["type"], | |||||
| "metadata": provider["metadata"], | |||||
| } | |||||
| def test_retrying_proxy_storage_metadata_provider_add(swh_storage, sample_data): | |||||
| """Standard metadata_provider_add works as before | |||||
| """ | """ | ||||
| provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
| provider_get = to_provider(provider) | |||||
| provider = swh_storage.metadata_provider_get_by(provider_get) | assert not swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
| assert not provider | |||||
| provider_id = swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
| assert provider_id | |||||
| actual_provider = swh_storage.metadata_provider_get(provider_id) | actual_authority = swh_storage.metadata_authority_get( | ||||
| assert actual_provider | authority["type"], authority["url"] | ||||
| actual_provider_id = actual_provider.pop("id") | ) | ||||
| assert actual_provider_id == provider_id | assert actual_authority == authority | ||||
| assert actual_provider == provider_get | |||||
| def test_retrying_proxy_storage_metadata_provider_add_with_retry( | def test_retrying_proxy_storage_metadata_authority_add_with_retry( | ||||
| swh_storage, sample_data, mocker, fake_hash_collision | swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
| provider_get = to_provider(provider) | |||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_provider_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("provider_id already inserted"), | psycopg2.IntegrityError("foo bar"), | ||||
| # ok then! | # ok then! | ||||
| "provider_id", | None, | ||||
| ] | ] | ||||
| from swh.storage.retry import RetryingProxyStorage # noqa | |||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| "swh.storage.retry.RetryingProxyStorage" ".metadata_provider_add.retry.sleep" | "swh.storage.retry.RetryingProxyStorage" ".metadata_authority_add.retry.sleep" | ||||
ardumont: can you please merge the string, that's black doing some joke on us? | |||||
| ) | ) | ||||
| provider = swh_storage.metadata_provider_get_by(provider_get) | assert not swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
| assert not provider | |||||
| provider_id = swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
| assert provider_id == "provider_id" | |||||
| provider_arg_names = ("provider_name", "provider_type", "provider_url", "metadata") | authority_arg_names = ("type", "url", "metadata") | ||||
| provider_args = [provider_get[key] for key in provider_arg_names] | authority_args = [authority[key] for key in authority_arg_names] | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call(*provider_args), call(*provider_args), call(*provider_args),] | [call(*authority_args), call(*authority_args), call(*authority_args),] | ||||
| ) | ) | ||||
| assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
| def test_retrying_proxy_swh_storage_metadata_provider_add_failure( | def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_provider_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add provider_id always!" | "Refuse to add authority_id always!" | ||||
| ) | ) | ||||
| provider = sample_data["provider"][0] | authority = sample_data["authority"][0] | ||||
| provider_get = to_provider(provider) | |||||
| provider_id = swh_storage.metadata_provider_get_by(provider_get) | swh_storage.metadata_authority_get(authority["type"], authority["url"]) | ||||
| assert not provider_id | |||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.metadata_provider_add(**provider_get) | swh_storage.metadata_authority_add(**authority) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_origin_metadata_add(swh_storage, sample_data): | def test_retrying_proxy_storage_origin_metadata_add(swh_storage, sample_data): | ||||
| """Standard origin_metadata_add works as before | """Standard origin_metadata_add works as before | ||||
| """ | """ | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
| swh_storage.origin_add_one(origin) | swh_storage.metadata_authority_add(**sample_data["authority"][0]) | ||||
| provider_get = to_provider(ori_meta["provider"]) | swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0]) | ||||
| provider_id = swh_storage.metadata_provider_add(**provider_get) | |||||
| origin_metadata = swh_storage.origin_metadata_get_by(origin["url"]) | origin_metadata = swh_storage.origin_metadata_get( | ||||
| ori_meta["origin_url"], ori_meta["authority"] | |||||
| ) | |||||
| assert not origin_metadata | assert not origin_metadata | ||||
| swh_storage.origin_metadata_add( | swh_storage.origin_metadata_add(**ori_meta) | ||||
| origin["url"], | |||||
| ori_meta["discovery_date"], | |||||
| provider_id, | |||||
| ori_meta["tool"], | |||||
| ori_meta["metadata"], | |||||
| ) | |||||
| origin_metadata = swh_storage.origin_metadata_get_by(origin["url"]) | origin_metadata = swh_storage.origin_metadata_get( | ||||
| ori_meta["origin_url"], ori_meta["authority"] | |||||
| ) | |||||
| assert origin_metadata | assert origin_metadata | ||||
| def test_retrying_proxy_storage_origin_metadata_add_with_retry( | def test_retrying_proxy_storage_origin_metadata_add_with_retry( | ||||
| swh_storage, sample_data, mocker, fake_hash_collision | swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
| swh_storage.origin_add_one(origin) | swh_storage.metadata_authority_add(**sample_data["authority"][0]) | ||||
| provider_get = to_provider(ori_meta["provider"]) | swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0]) | ||||
| provider_id = swh_storage.metadata_provider_add(**provider_get) | |||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("provider_id already inserted"), | psycopg2.IntegrityError("foo bar"), | ||||
| # ok then! | # ok then! | ||||
| None, | None, | ||||
| ] | ] | ||||
| mock_sleep = mocker.patch( | mock_sleep = mocker.patch( | ||||
| "swh.storage.retry.RetryingProxyStorage" ".origin_metadata_add.retry.sleep" | "swh.storage.retry.RetryingProxyStorage" ".origin_metadata_add.retry.sleep" | ||||
| ) | ) | ||||
| url = origin["url"] | |||||
| ts = ori_meta["discovery_date"] | |||||
| tool_id = ori_meta["tool"] | |||||
| metadata = ori_meta["metadata"] | |||||
| # No exception raised as insertion finally came through | # No exception raised as insertion finally came through | ||||
| swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, metadata) | swh_storage.origin_metadata_add(**ori_meta) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [ # 3 calls, as long as error raised | [ # 3 calls, as long as error raised | ||||
| call(url, ts, provider_id, tool_id, metadata), | call( | ||||
| call(url, ts, provider_id, tool_id, metadata), | ori_meta["origin_url"], | ||||
| call(url, ts, provider_id, tool_id, metadata), | ori_meta["discovery_date"], | ||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| call( | |||||
| ori_meta["origin_url"], | |||||
| ori_meta["discovery_date"], | |||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| call( | |||||
| ori_meta["origin_url"], | |||||
| ori_meta["discovery_date"], | |||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| ] | ] | ||||
| ) | ) | ||||
| assert mock_sleep.call_count == 2 | assert mock_sleep.call_count == 2 | ||||
| def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| origin = ori_meta["origin"] | swh_storage.origin_add_one({"url": ori_meta["origin_url"]}) | ||||
| swh_storage.origin_add_one(origin) | |||||
| url = origin["url"] | |||||
| ts = ori_meta["discovery_date"] | |||||
| provider_id = "provider_id" | |||||
| tool_id = ori_meta["tool"] | |||||
| metadata = ori_meta["metadata"] | |||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.origin_metadata_add(url, ts, provider_id, tool_id, metadata) | swh_storage.origin_metadata_add(**ori_meta) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | ||||
| """Standard origin_visit_update works as before | """Standard origin_visit_update works as before | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 389 Lines • Show Last 20 Lines | |||||
can you please merge the string, that's black doing some joke on us?