Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| Show All 29 Lines | def monkeypatch_sleep(monkeypatch, swh_storage): | ||||
| for method_name, method in RetryingProxyStorage.__dict__.items(): | for method_name, method in RetryingProxyStorage.__dict__.items(): | ||||
| if "_add" in method_name or "_update" in method_name: | if "_add" in method_name or "_update" in method_name: | ||||
| monkeypatch.setattr(method.retry, "sleep", lambda x: None) | monkeypatch.setattr(method.retry, "sleep", lambda x: None) | ||||
| return monkeypatch | return monkeypatch | ||||
| @pytest.fixture | @pytest.fixture | ||||
| def fake_hash_collision(sample_data_model): | def fake_hash_collision(sample_data): | ||||
| return HashCollision("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a", []) | return HashCollision("sha1", "38762cf7f55934b34d179ae6a4c80cadccbb7f0a", []) | ||||
| @pytest.fixture | @pytest.fixture | ||||
| def swh_storage_backend_config(): | def swh_storage_backend_config(): | ||||
| yield { | yield { | ||||
| "cls": "pipeline", | "cls": "pipeline", | ||||
| "steps": [{"cls": "retry"}, {"cls": "memory"},], | "steps": [{"cls": "retry"}, {"cls": "memory"},], | ||||
| } | } | ||||
| def test_retrying_proxy_storage_content_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_content_add(swh_storage, sample_data): | ||||
| """Standard content_add works as before | """Standard content_add works as before | ||||
| """ | """ | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| s = swh_storage.content_add([sample_content]) | s = swh_storage.content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| "content:add": 1, | "content:add": 1, | ||||
| "content:add:bytes": sample_content.length, | "content:add:bytes": sample_content.length, | ||||
| } | } | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert content["sha1"] == sample_content.sha1 | assert content["sha1"] == sample_content.sha1 | ||||
| def test_retrying_proxy_storage_content_add_with_retry( | def test_retrying_proxy_storage_content_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("content already inserted"), | psycopg2.IntegrityError("content already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"content:add": 1}, | {"content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| s = swh_storage.content_add([sample_content]) | s = swh_storage.content_add([sample_content]) | ||||
| assert s == {"content:add": 1} | assert s == {"content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_content]), call([sample_content]), call([sample_content]),] | [call([sample_content]), call([sample_content]), call([sample_content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_content_add_failure( | def test_retrying_proxy_swh_storage_content_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add content always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add content always!") | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.content_add([sample_content]) | swh_storage.content_add([sample_content]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data_model): | def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | ||||
| """Standard content_add_metadata works as before | """Standard content_add_metadata works as before | ||||
| """ | """ | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| pk = content.sha1 | pk = content.sha1 | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert not content_metadata[pk] | assert not content_metadata[pk] | ||||
| s = swh_storage.content_add_metadata([content]) | s = swh_storage.content_add_metadata([content]) | ||||
| assert s == { | assert s == { | ||||
| "content:add": 1, | "content:add": 1, | ||||
| } | } | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert len(content_metadata[pk]) == 1 | assert len(content_metadata[pk]) == 1 | ||||
| assert content_metadata[pk][0]["sha1"] == pk | assert content_metadata[pk][0]["sha1"] == pk | ||||
| def test_retrying_proxy_storage_content_add_metadata_with_retry( | def test_retrying_proxy_storage_content_add_metadata_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("content_metadata already inserted"), | psycopg2.IntegrityError("content_metadata already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"content:add": 1}, | {"content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| s = swh_storage.content_add_metadata([content]) | s = swh_storage.content_add_metadata([content]) | ||||
| assert s == {"content:add": 1} | assert s == {"content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([content]), call([content]), call([content]),] | [call([content]), call([content]), call([content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_content_add_metadata_failure( | def test_retrying_proxy_swh_storage_content_add_metadata_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add content_metadata!" | "Refuse to add content_metadata!" | ||||
| ) | ) | ||||
| sample_content = sample_data_model["content"][0] | sample_content = sample_data["content"][0] | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| pk = content.sha1 | pk = content.sha1 | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert not content_metadata[pk] | assert not content_metadata[pk] | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.content_add_metadata([content]) | swh_storage.content_add_metadata([content]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_skipped_content_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_skipped_content_add(swh_storage, sample_data): | ||||
| """Standard skipped_content_add works as before | """Standard skipped_content_add works as before | ||||
| """ | """ | ||||
| sample_content = sample_data_model["skipped_content"][0] | sample_content = sample_data["skipped_content"][0] | ||||
| sample_content_dict = sample_content.to_dict() | sample_content_dict = sample_content.to_dict() | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| s = swh_storage.skipped_content_add([sample_content]) | s = swh_storage.skipped_content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| "skipped_content:add": 1, | "skipped_content:add": 1, | ||||
| } | } | ||||
| skipped_content = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_content = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_content) == 0 | assert len(skipped_content) == 0 | ||||
| def test_retrying_proxy_storage_skipped_content_add_with_retry( | def test_retrying_proxy_storage_skipped_content_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # 1st & 2nd try goes ko | # 1st & 2nd try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| psycopg2.IntegrityError("skipped_content already inserted"), | psycopg2.IntegrityError("skipped_content already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"skipped_content:add": 1}, | {"skipped_content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data_model["skipped_content"][0] | sample_content = sample_data["skipped_content"][0] | ||||
| s = swh_storage.skipped_content_add([sample_content]) | s = swh_storage.skipped_content_add([sample_content]) | ||||
| assert s == {"skipped_content:add": 1} | assert s == {"skipped_content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_content]), call([sample_content]), call([sample_content]),] | [call([sample_content]), call([sample_content]), call([sample_content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_skipped_content_add_failure( | def test_retrying_proxy_swh_storage_skipped_content_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add content_metadata!" | "Refuse to add content_metadata!" | ||||
| ) | ) | ||||
| sample_content = sample_data_model["skipped_content"][0] | sample_content = sample_data["skipped_content"][0] | ||||
| sample_content_dict = sample_content.to_dict() | sample_content_dict = sample_content.to_dict() | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.skipped_content_add([sample_content]) | swh_storage.skipped_content_add([sample_content]) | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data_model): | def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | ||||
| """Standard origin_visit_add works as before | """Standard origin_visit_add works as before | ||||
| """ | """ | ||||
| origin = sample_data_model["origin"][0] | origin = sample_data["origin"][0] | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| origins = list(swh_storage.origin_visit_get(origin.url)) | origins = list(swh_storage.origin_visit_get(origin.url)) | ||||
| assert not origins | assert not origins | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="hg") | visit = OriginVisit(origin=origin.url, date=date_visit1, type="hg") | ||||
| origin_visit = swh_storage.origin_visit_add([visit])[0] | origin_visit = swh_storage.origin_visit_add([visit])[0] | ||||
| assert origin_visit.origin == origin.url | assert origin_visit.origin == origin.url | ||||
| assert isinstance(origin_visit.visit, int) | assert isinstance(origin_visit.visit, int) | ||||
| origin_visit = next(swh_storage.origin_visit_get(origin.url)) | origin_visit = next(swh_storage.origin_visit_get(origin.url)) | ||||
| assert origin_visit["origin"] == origin.url | assert origin_visit["origin"] == origin.url | ||||
| assert isinstance(origin_visit["visit"], int) | assert isinstance(origin_visit["visit"], int) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_retry( | def test_retrying_proxy_swh_storage_origin_visit_add_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| origin = sample_data_model["origin"][1] | origin = sample_data["origin"][1] | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="git") | visit = OriginVisit(origin=origin.url, date=date_visit1, type="git") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| Show All 9 Lines | ): | ||||
| assert r == [visit] | assert r == [visit] | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([visit]), call([visit]), call([visit]),] | [call([visit]), call([visit]), call([visit]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_failure( | def test_retrying_proxy_swh_storage_origin_visit_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | ||||
| origin = sample_data_model["origin"][0] | origin = sample_data["origin"][0] | ||||
| origins = list(swh_storage.origin_visit_get(origin.url)) | origins = list(swh_storage.origin_visit_get(origin.url)) | ||||
| assert not origins | assert not origins | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="svn",) | visit = OriginVisit(origin=origin.url, date=date_visit1, type="svn",) | ||||
| swh_storage.origin_visit_add([visit]) | swh_storage.origin_visit_add([visit]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([visit]),] | [call([visit]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | ||||
| """Standard metadata_fetcher_add works as before | """Standard metadata_fetcher_add works as before | ||||
| """ | """ | ||||
| fetcher = sample_data_model["fetcher"][0] | fetcher = sample_data["fetcher"][0] | ||||
| metadata_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | metadata_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert not metadata_fetcher | assert not metadata_fetcher | ||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert actual_fetcher == fetcher | assert actual_fetcher == fetcher | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| fetcher = sample_data_model["fetcher"][0] | fetcher = sample_data["fetcher"][0] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("metadata_fetcher already inserted"), | psycopg2.IntegrityError("metadata_fetcher already inserted"), | ||||
| # ok then! | # ok then! | ||||
| [fetcher], | [fetcher], | ||||
| ] | ] | ||||
| actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert not actual_fetcher | assert not actual_fetcher | ||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([fetcher]), call([fetcher]), call([fetcher]),] | [call([fetcher]), call([fetcher]), call([fetcher]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure( | def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add metadata_fetcher always!" | "Refuse to add metadata_fetcher always!" | ||||
| ) | ) | ||||
| fetcher = sample_data_model["fetcher"][0] | fetcher = sample_data["fetcher"][0] | ||||
| actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert not actual_fetcher | assert not actual_fetcher | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data): | ||||
| """Standard metadata_authority_add works as before | """Standard metadata_authority_add works as before | ||||
| """ | """ | ||||
| authority = sample_data_model["authority"][0] | authority = sample_data["authority"][0] | ||||
| assert not swh_storage.metadata_authority_get(authority.type, authority.url) | assert not swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| actual_authority = swh_storage.metadata_authority_get(authority.type, authority.url) | actual_authority = swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| assert actual_authority == authority | assert actual_authority == authority | ||||
| def test_retrying_proxy_storage_metadata_authority_add_with_retry( | def test_retrying_proxy_storage_metadata_authority_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| authority = sample_data_model["authority"][0] | authority = sample_data["authority"][0] | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("foo bar"), | psycopg2.IntegrityError("foo bar"), | ||||
| # ok then! | # ok then! | ||||
| None, | None, | ||||
| ] | ] | ||||
| assert not swh_storage.metadata_authority_get(authority.type, authority.url) | assert not swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([authority]), call([authority]), call([authority])] | [call([authority]), call([authority]), call([authority])] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | def test_retrying_proxy_swh_storage_metadata_authority_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add authority_id always!" | "Refuse to add authority_id always!" | ||||
| ) | ) | ||||
| authority = sample_data_model["authority"][0] | authority = sample_data["authority"][0] | ||||
| swh_storage.metadata_authority_get(authority.type, authority.url) | swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_object_metadata_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_object_metadata_add(swh_storage, sample_data): | ||||
| """Standard object_metadata_add works as before | """Standard object_metadata_add works as before | ||||
| """ | """ | ||||
| origin = sample_data_model["origin"][0] | origin = sample_data["origin"][0] | ||||
| ori_meta = sample_data_model["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| swh_storage.metadata_authority_add([sample_data_model["authority"][0]]) | swh_storage.metadata_authority_add([sample_data["authority"][0]]) | ||||
| swh_storage.metadata_fetcher_add([sample_data_model["fetcher"][0]]) | swh_storage.metadata_fetcher_add([sample_data["fetcher"][0]]) | ||||
| origin_metadata = swh_storage.object_metadata_get( | origin_metadata = swh_storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata["next_page_token"] is None | assert origin_metadata["next_page_token"] is None | ||||
| assert not origin_metadata["results"] | assert not origin_metadata["results"] | ||||
| swh_storage.object_metadata_add([ori_meta]) | swh_storage.object_metadata_add([ori_meta]) | ||||
| origin_metadata = swh_storage.object_metadata_get( | origin_metadata = swh_storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata | assert origin_metadata | ||||
| def test_retrying_proxy_storage_object_metadata_add_with_retry( | def test_retrying_proxy_storage_object_metadata_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| origin = sample_data_model["origin"][0] | origin = sample_data["origin"][0] | ||||
| ori_meta = sample_data_model["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| swh_storage.metadata_authority_add([sample_data_model["authority"][0]]) | swh_storage.metadata_authority_add([sample_data["authority"][0]]) | ||||
| swh_storage.metadata_fetcher_add([sample_data_model["fetcher"][0]]) | swh_storage.metadata_fetcher_add([sample_data["fetcher"][0]]) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| Show All 10 Lines | mock_memory.assert_has_calls( | ||||
| call([ori_meta]), | call([ori_meta]), | ||||
| call([ori_meta]), | call([ori_meta]), | ||||
| call([ori_meta]), | call([ori_meta]), | ||||
| ] | ] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_object_metadata_add_failure( | def test_retrying_proxy_swh_storage_object_metadata_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | ||||
| origin = sample_data_model["origin"][0] | origin = sample_data["origin"][0] | ||||
| ori_meta = sample_data_model["origin_metadata"][0] | ori_meta = sample_data["origin_metadata"][0] | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.object_metadata_add([ori_meta]) | swh_storage.object_metadata_add([ori_meta]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_directory_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | ||||
| """Standard directory_add works as before | """Standard directory_add works as before | ||||
| """ | """ | ||||
| sample_dir = sample_data_model["directory"][0] | sample_dir = sample_data["directory"][0] | ||||
| directory = swh_storage.directory_get_random() # no directory | directory = swh_storage.directory_get_random() # no directory | ||||
| assert not directory | assert not directory | ||||
| s = swh_storage.directory_add([sample_dir]) | s = swh_storage.directory_add([sample_dir]) | ||||
| assert s == { | assert s == { | ||||
| "directory:add": 1, | "directory:add": 1, | ||||
| } | } | ||||
| directory_id = swh_storage.directory_get_random() # only 1 | directory_id = swh_storage.directory_get_random() # only 1 | ||||
| assert directory_id == sample_dir.id | assert directory_id == sample_dir.id | ||||
| def test_retrying_proxy_storage_directory_add_with_retry( | def test_retrying_proxy_storage_directory_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("directory already inserted"), | psycopg2.IntegrityError("directory already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"directory:add": 1}, | {"directory:add": 1}, | ||||
| ] | ] | ||||
| sample_dir = sample_data_model["directory"][1] | sample_dir = sample_data["directory"][1] | ||||
| directory_id = swh_storage.directory_get_random() # no directory | directory_id = swh_storage.directory_get_random() # no directory | ||||
| assert not directory_id | assert not directory_id | ||||
| s = swh_storage.directory_add([sample_dir]) | s = swh_storage.directory_add([sample_dir]) | ||||
| assert s == { | assert s == { | ||||
| "directory:add": 1, | "directory:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_dir]), call([sample_dir]), call([sample_dir]),] | [call([sample_dir]), call([sample_dir]), call([sample_dir]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_directory_add_failure( | def test_retrying_proxy_swh_storage_directory_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add directory always!" | "Refuse to add directory always!" | ||||
| ) | ) | ||||
| sample_dir = sample_data_model["directory"][0] | sample_dir = sample_data["directory"][0] | ||||
| directory_id = swh_storage.directory_get_random() # no directory | directory_id = swh_storage.directory_get_random() # no directory | ||||
| assert not directory_id | assert not directory_id | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.directory_add([sample_dir]) | swh_storage.directory_add([sample_dir]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_revision_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_revision_add(swh_storage, sample_data): | ||||
| """Standard revision_add works as before | """Standard revision_add works as before | ||||
| """ | """ | ||||
| sample_rev = sample_data_model["revision"][0] | sample_rev = sample_data["revision"][0] | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| s = swh_storage.revision_add([sample_rev]) | s = swh_storage.revision_add([sample_rev]) | ||||
| assert s == { | assert s == { | ||||
| "revision:add": 1, | "revision:add": 1, | ||||
| } | } | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert revision["id"] == sample_rev.id | assert revision["id"] == sample_rev.id | ||||
| def test_retrying_proxy_storage_revision_add_with_retry( | def test_retrying_proxy_storage_revision_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("revision already inserted"), | psycopg2.IntegrityError("revision already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"revision:add": 1}, | {"revision:add": 1}, | ||||
| ] | ] | ||||
| sample_rev = sample_data_model["revision"][0] | sample_rev = sample_data["revision"][0] | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| s = swh_storage.revision_add([sample_rev]) | s = swh_storage.revision_add([sample_rev]) | ||||
| assert s == { | assert s == { | ||||
| "revision:add": 1, | "revision:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_rev]), call([sample_rev]), call([sample_rev]),] | [call([sample_rev]), call([sample_rev]), call([sample_rev]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_revision_add_failure( | def test_retrying_proxy_swh_storage_revision_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add revision always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add revision always!") | ||||
| sample_rev = sample_data_model["revision"][0] | sample_rev = sample_data["revision"][0] | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.revision_add([sample_rev]) | swh_storage.revision_add([sample_rev]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_release_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_release_add(swh_storage, sample_data): | ||||
| """Standard release_add works as before | """Standard release_add works as before | ||||
| """ | """ | ||||
| sample_rel = sample_data_model["release"][0] | sample_rel = sample_data["release"][0] | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| s = swh_storage.release_add([sample_rel]) | s = swh_storage.release_add([sample_rel]) | ||||
| assert s == { | assert s == { | ||||
| "release:add": 1, | "release:add": 1, | ||||
| } | } | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert release["id"] == sample_rel.id | assert release["id"] == sample_rel.id | ||||
| def test_retrying_proxy_storage_release_add_with_retry( | def test_retrying_proxy_storage_release_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("release already inserted"), | psycopg2.IntegrityError("release already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"release:add": 1}, | {"release:add": 1}, | ||||
| ] | ] | ||||
| sample_rel = sample_data_model["release"][0] | sample_rel = sample_data["release"][0] | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| s = swh_storage.release_add([sample_rel]) | s = swh_storage.release_add([sample_rel]) | ||||
| assert s == { | assert s == { | ||||
| "release:add": 1, | "release:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_rel]), call([sample_rel]), call([sample_rel]),] | [call([sample_rel]), call([sample_rel]), call([sample_rel]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_release_add_failure( | def test_retrying_proxy_swh_storage_release_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add release always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add release always!") | ||||
| sample_rel = sample_data_model["release"][0] | sample_rel = sample_data["release"][0] | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.release_add([sample_rel]) | swh_storage.release_add([sample_rel]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data_model): | def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data): | ||||
| """Standard snapshot_add works as before | """Standard snapshot_add works as before | ||||
| """ | """ | ||||
| sample_snap = sample_data_model["snapshot"][0] | sample_snap = sample_data["snapshot"][0] | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| s = swh_storage.snapshot_add([sample_snap]) | s = swh_storage.snapshot_add([sample_snap]) | ||||
| assert s == { | assert s == { | ||||
| "snapshot:add": 1, | "snapshot:add": 1, | ||||
| } | } | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert snapshot["id"] == sample_snap.id | assert snapshot["id"] == sample_snap.id | ||||
| def test_retrying_proxy_storage_snapshot_add_with_retry( | def test_retrying_proxy_storage_snapshot_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data_model, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("snapshot already inserted"), | psycopg2.IntegrityError("snapshot already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"snapshot:add": 1}, | {"snapshot:add": 1}, | ||||
| ] | ] | ||||
| sample_snap = sample_data_model["snapshot"][0] | sample_snap = sample_data["snapshot"][0] | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| s = swh_storage.snapshot_add([sample_snap]) | s = swh_storage.snapshot_add([sample_snap]) | ||||
| assert s == { | assert s == { | ||||
| "snapshot:add": 1, | "snapshot:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_snap]), call([sample_snap]), call([sample_snap]),] | [call([sample_snap]), call([sample_snap]), call([sample_snap]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_snapshot_add_failure( | def test_retrying_proxy_swh_storage_snapshot_add_failure( | ||||
| swh_storage, sample_data_model, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add snapshot always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add snapshot always!") | ||||
| sample_snap = sample_data_model["snapshot"][0] | sample_snap = sample_data["snapshot"][0] | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.snapshot_add([sample_snap]) | swh_storage.snapshot_add([sample_snap]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||