Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from unittest.mock import call | from unittest.mock import call | ||||
| import psycopg2 | import psycopg2 | ||||
| import pytest | import pytest | ||||
| from swh.model.model import OriginVisit | from swh.model.model import ( | ||||
| OriginVisit, | |||||
| MetadataTargetType, | |||||
| ) | |||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from swh.storage.exc import HashCollision, StorageArgumentException | from swh.storage.exc import HashCollision, StorageArgumentException | ||||
| from .storage_data import date_visit1 | from .storage_data import date_visit1 | ||||
| @pytest.fixture | @pytest.fixture | ||||
| ▲ Show 20 Lines • Show All 383 Lines • ▼ Show 20 Lines | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="svn",) | visit = OriginVisit(origin=origin.url, date=date_visit1, type="svn",) | ||||
| swh_storage.origin_visit_add([visit]) | swh_storage.origin_visit_add([visit]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([visit]),] | [call([visit]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage_validate, sample_data): | def test_retrying_proxy_storage_metadata_fetcher_add( | ||||
| swh_storage_validate, sample_data_model | |||||
| ): | |||||
| """Standard metadata_fetcher_add works as before | """Standard metadata_fetcher_add works as before | ||||
| """ | """ | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data_model["fetcher"][0] | ||||
| metadata_fetcher = swh_storage_validate.metadata_fetcher_get( | metadata_fetcher = swh_storage_validate.metadata_fetcher_get( | ||||
| fetcher["name"], fetcher["version"] | fetcher.name, fetcher.version | ||||
| ) | ) | ||||
| assert not metadata_fetcher | assert not metadata_fetcher | ||||
| swh_storage_validate.metadata_fetcher_add(**fetcher) | swh_storage_validate.metadata_fetcher_add([fetcher]) | ||||
| actual_fetcher = swh_storage_validate.metadata_fetcher_get( | actual_fetcher = swh_storage_validate.metadata_fetcher_get( | ||||
| fetcher["name"], fetcher["version"] | fetcher.name, fetcher.version | ||||
| ) | ) | ||||
| assert actual_fetcher == fetcher | assert actual_fetcher == fetcher | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision | monkeypatch_sleep, | ||||
| swh_storage_validate, | |||||
| sample_data_model, | |||||
| mocker, | |||||
| fake_hash_collision, | |||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data_model["fetcher"][0] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("metadata_fetcher already inserted"), | psycopg2.IntegrityError("metadata_fetcher already inserted"), | ||||
| # ok then! | # ok then! | ||||
| [fetcher], | [fetcher], | ||||
| ] | ] | ||||
| actual_fetcher = swh_storage_validate.metadata_fetcher_get( | actual_fetcher = swh_storage_validate.metadata_fetcher_get( | ||||
| fetcher["name"], fetcher["version"] | fetcher.name, fetcher.version | ||||
| ) | ) | ||||
| assert not actual_fetcher | assert not actual_fetcher | ||||
| swh_storage_validate.metadata_fetcher_add(**fetcher) | swh_storage_validate.metadata_fetcher_add([fetcher]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [ | [call([fetcher]), call([fetcher]), call([fetcher]),] | ||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| call(fetcher["name"], fetcher["version"], fetcher["metadata"]), | |||||
| ] | |||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure( | def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure( | ||||
| swh_storage_validate, sample_data, mocker | swh_storage_validate, sample_data_model, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add metadata_fetcher always!" | "Refuse to add metadata_fetcher always!" | ||||
| ) | ) | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data_model["fetcher"][0] | ||||
| actual_fetcher = swh_storage_validate.metadata_fetcher_get( | actual_fetcher = swh_storage_validate.metadata_fetcher_get( | ||||
| fetcher["name"], fetcher["version"] | fetcher.name, fetcher.version | ||||
| ) | ) | ||||
| assert not actual_fetcher | assert not actual_fetcher | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage_validate.metadata_fetcher_add(**fetcher) | swh_storage_validate.metadata_fetcher_add([fetcher]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_metadata_authority_add( | def test_retrying_proxy_storage_metadata_authority_add( | ||||
| swh_storage_validate, sample_data | swh_storage_validate, sample_data_model | ||||
| ): | ): | ||||
| """Standard metadata_authority_add works as before | """Standard metadata_authority_add works as before | ||||
| """ | """ | ||||
| authority = sample_data["authority"][0] | authority = sample_data_model["authority"][0] | ||||
| assert not swh_storage_validate.metadata_authority_get( | assert not swh_storage_validate.metadata_authority_get( | ||||
| authority["type"], authority["url"] | authority.type, authority.url | ||||
| ) | ) | ||||
| swh_storage_validate.metadata_authority_add(**authority) | swh_storage_validate.metadata_authority_add([authority]) | ||||
| actual_authority = swh_storage_validate.metadata_authority_get( | actual_authority = swh_storage_validate.metadata_authority_get( | ||||
| authority["type"], authority["url"] | authority.type, authority.url | ||||
| ) | ) | ||||
| assert actual_authority == authority | assert actual_authority == authority | ||||
| def test_retrying_proxy_storage_metadata_authority_add_with_retry( | def test_retrying_proxy_storage_metadata_authority_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision | monkeypatch_sleep, | ||||
| swh_storage_validate, | |||||
| sample_data_model, | |||||
| mocker, | |||||
| fake_hash_collision, | |||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| authority = sample_data["authority"][0] | authority = sample_data_model["authority"][0] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("foo bar"), | psycopg2.IntegrityError("foo bar"), | ||||
| # ok then! | # ok then! | ||||
| None, | None, | ||||
| ] | ] | ||||
| assert not swh_storage_validate.metadata_authority_get( | assert not swh_storage_validate.metadata_authority_get( | ||||
| authority["type"], authority["url"] | authority.type, authority.url | ||||
| ) | ) | ||||
| swh_storage_validate.metadata_authority_add(**authority) | swh_storage_validate.metadata_authority_add([authority]) | ||||
| authority_arg_names = ("type", "url", "metadata") | |||||
| authority_args = [authority[key] for key in authority_arg_names] | |||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call(*authority_args), call(*authority_args), call(*authority_args),] | [call([authority]), call([authority]), call([authority])] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | ||||
| swh_storage_validate, sample_data, mocker | swh_storage_validate, sample_data_model, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add authority_id always!" | "Refuse to add authority_id always!" | ||||
| ) | ) | ||||
| authority = sample_data["authority"][0] | authority = sample_data_model["authority"][0] | ||||
| swh_storage_validate.metadata_authority_get(authority["type"], authority["url"]) | swh_storage_validate.metadata_authority_get(authority.type, authority.url) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage_validate.metadata_authority_add(**authority) | swh_storage_validate.metadata_authority_add([authority]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_origin_metadata_add(swh_storage_validate, sample_data): | def test_retrying_proxy_storage_object_metadata_add( | ||||
| """Standard origin_metadata_add works as before | swh_storage_validate, sample_data_model | ||||
| ): | |||||
| """Standard object_metadata_add works as before | |||||
| """ | """ | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data_model["origin_metadata"][0] | ||||
| swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]}) | swh_storage_validate.origin_add_one({"url": ori_meta.id}) | ||||
| swh_storage_validate.metadata_authority_add(**sample_data["authority"][0]) | swh_storage_validate.metadata_authority_add([sample_data_model["authority"][0]]) | ||||
| swh_storage_validate.metadata_fetcher_add(**sample_data["fetcher"][0]) | swh_storage_validate.metadata_fetcher_add([sample_data_model["fetcher"][0]]) | ||||
| origin_metadata = swh_storage_validate.origin_metadata_get( | origin_metadata = swh_storage_validate.object_metadata_get( | ||||
| ori_meta["origin_url"], ori_meta["authority"] | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata["next_page_token"] is None | assert origin_metadata["next_page_token"] is None | ||||
| assert not origin_metadata["results"] | assert not origin_metadata["results"] | ||||
| swh_storage_validate.origin_metadata_add(**ori_meta) | swh_storage_validate.object_metadata_add([ori_meta]) | ||||
| origin_metadata = swh_storage_validate.origin_metadata_get( | origin_metadata = swh_storage_validate.object_metadata_get( | ||||
| ori_meta["origin_url"], ori_meta["authority"] | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata | assert origin_metadata | ||||
| def test_retrying_proxy_storage_origin_metadata_add_with_retry( | def test_retrying_proxy_storage_object_metadata_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision | monkeypatch_sleep, | ||||
| swh_storage_validate, | |||||
| sample_data_model, | |||||
| mocker, | |||||
| fake_hash_collision, | |||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data_model["origin_metadata"][0] | ||||
| swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]}) | swh_storage_validate.origin_add_one({"url": ori_meta.id}) | ||||
| swh_storage_validate.metadata_authority_add(**sample_data["authority"][0]) | swh_storage_validate.metadata_authority_add([sample_data_model["authority"][0]]) | ||||
| swh_storage_validate.metadata_fetcher_add(**sample_data["fetcher"][0]) | swh_storage_validate.metadata_fetcher_add([sample_data_model["fetcher"][0]]) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("foo bar"), | psycopg2.IntegrityError("foo bar"), | ||||
| # ok then! | # ok then! | ||||
| None, | None, | ||||
| ] | ] | ||||
| # No exception raised as insertion finally came through | # No exception raised as insertion finally came through | ||||
| swh_storage_validate.origin_metadata_add(**ori_meta) | swh_storage_validate.object_metadata_add([ori_meta]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [ # 3 calls, as long as error raised | [ # 3 calls, as long as error raised | ||||
| call( | call([ori_meta]), | ||||
| ori_meta["origin_url"], | call([ori_meta]), | ||||
| ori_meta["discovery_date"], | call([ori_meta]), | ||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| call( | |||||
| ori_meta["origin_url"], | |||||
| ori_meta["discovery_date"], | |||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| call( | |||||
| ori_meta["origin_url"], | |||||
| ori_meta["discovery_date"], | |||||
| ori_meta["authority"], | |||||
| ori_meta["fetcher"], | |||||
| ori_meta["format"], | |||||
| ori_meta["metadata"], | |||||
| ), | |||||
| ] | ] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_origin_metadata_add_failure( | def test_retrying_proxy_swh_storage_object_metadata_add_failure( | ||||
| swh_storage_validate, sample_data, mocker | swh_storage_validate, sample_data_model, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.origin_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data_model["origin_metadata"][0] | ||||
| swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]}) | swh_storage_validate.origin_add_one({"url": ori_meta.id}) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage_validate.origin_metadata_add(**ori_meta) | swh_storage_validate.object_metadata_add([ori_meta]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_directory_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_directory_add(swh_storage, sample_data_model): | ||||
| """Standard directory_add works as before | """Standard directory_add works as before | ||||
| """ | """ | ||||
| ▲ Show 20 Lines • Show All 272 Lines • Show Last 20 Lines | |||||