Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import attr | import attr | ||||
| from unittest.mock import call | from unittest.mock import call | ||||
| import psycopg2 | import psycopg2 | ||||
| import pytest | import pytest | ||||
| from swh.model.model import ( | from swh.model.model import MetadataTargetType | ||||
| OriginVisit, | |||||
| MetadataTargetType, | |||||
| ) | |||||
| from swh.storage.exc import HashCollision, StorageArgumentException | from swh.storage.exc import HashCollision, StorageArgumentException | ||||
| from .storage_data import date_visit1 | |||||
| @pytest.fixture | @pytest.fixture | ||||
| def monkeypatch_sleep(monkeypatch, swh_storage): | def monkeypatch_sleep(monkeypatch, swh_storage): | ||||
| """In test context, we don't want to wait, make test faster | """In test context, we don't want to wait, make test faster | ||||
| """ | """ | ||||
| from swh.storage.retry import RetryingProxyStorage | from swh.storage.retry import RetryingProxyStorage | ||||
| Show All 16 Lines | yield { | ||||
| "steps": [{"cls": "retry"}, {"cls": "memory"},], | "steps": [{"cls": "retry"}, {"cls": "memory"},], | ||||
| } | } | ||||
| def test_retrying_proxy_storage_content_add(swh_storage, sample_data): | def test_retrying_proxy_storage_content_add(swh_storage, sample_data): | ||||
| """Standard content_add works as before | """Standard content_add works as before | ||||
| """ | """ | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| s = swh_storage.content_add([sample_content]) | s = swh_storage.content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| "content:add": 1, | "content:add": 1, | ||||
| "content:add:bytes": sample_content.length, | "content:add:bytes": sample_content.length, | ||||
| } | } | ||||
| Show All 13 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("content already inserted"), | psycopg2.IntegrityError("content already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"content:add": 1}, | {"content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| s = swh_storage.content_add([sample_content]) | s = swh_storage.content_add([sample_content]) | ||||
| assert s == {"content:add": 1} | assert s == {"content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_content]), call([sample_content]), call([sample_content]),] | [call([sample_content]), call([sample_content]), call([sample_content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_content_add_failure( | def test_retrying_proxy_swh_storage_content_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.content_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add content always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add content always!") | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = next(swh_storage.content_get([sample_content.sha1])) | content = next(swh_storage.content_get([sample_content.sha1])) | ||||
| assert not content | assert not content | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.content_add([sample_content]) | swh_storage.content_add([sample_content]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | def test_retrying_proxy_storage_content_add_metadata(swh_storage, sample_data): | ||||
| """Standard content_add_metadata works as before | """Standard content_add_metadata works as before | ||||
| """ | """ | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| pk = content.sha1 | pk = content.sha1 | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert not content_metadata[pk] | assert not content_metadata[pk] | ||||
| s = swh_storage.content_add_metadata([content]) | s = swh_storage.content_add_metadata([content]) | ||||
| assert s == { | assert s == { | ||||
| Show All 18 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("content_metadata already inserted"), | psycopg2.IntegrityError("content_metadata already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"content:add": 1}, | {"content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| s = swh_storage.content_add_metadata([content]) | s = swh_storage.content_add_metadata([content]) | ||||
| assert s == {"content:add": 1} | assert s == {"content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([content]), call([content]), call([content]),] | [call([content]), call([content]), call([content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_content_add_metadata_failure( | def test_retrying_proxy_swh_storage_content_add_metadata_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | "swh.storage.in_memory.InMemoryStorage.content_add_metadata" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add content_metadata!" | "Refuse to add content_metadata!" | ||||
| ) | ) | ||||
| sample_content = sample_data["content"][0] | sample_content = sample_data.content | ||||
| content = attr.evolve(sample_content, data=None) | content = attr.evolve(sample_content, data=None) | ||||
| pk = content.sha1 | pk = content.sha1 | ||||
| content_metadata = swh_storage.content_get_metadata([pk]) | content_metadata = swh_storage.content_get_metadata([pk]) | ||||
| assert not content_metadata[pk] | assert not content_metadata[pk] | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.content_add_metadata([content]) | swh_storage.content_add_metadata([content]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_skipped_content_add(swh_storage, sample_data): | def test_retrying_proxy_storage_skipped_content_add(swh_storage, sample_data): | ||||
| """Standard skipped_content_add works as before | """Standard skipped_content_add works as before | ||||
| """ | """ | ||||
| sample_content = sample_data["skipped_content"][0] | sample_content = sample_data.skipped_content | ||||
| sample_content_dict = sample_content.to_dict() | sample_content_dict = sample_content.to_dict() | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| s = swh_storage.skipped_content_add([sample_content]) | s = swh_storage.skipped_content_add([sample_content]) | ||||
| assert s == { | assert s == { | ||||
| "skipped_content:add": 1, | "skipped_content:add": 1, | ||||
| Show All 15 Lines | ): | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # 1st & 2nd try goes ko | # 1st & 2nd try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| psycopg2.IntegrityError("skipped_content already inserted"), | psycopg2.IntegrityError("skipped_content already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"skipped_content:add": 1}, | {"skipped_content:add": 1}, | ||||
| ] | ] | ||||
| sample_content = sample_data["skipped_content"][0] | sample_content = sample_data.skipped_content | ||||
| s = swh_storage.skipped_content_add([sample_content]) | s = swh_storage.skipped_content_add([sample_content]) | ||||
| assert s == {"skipped_content:add": 1} | assert s == {"skipped_content:add": 1} | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_content]), call([sample_content]), call([sample_content]),] | [call([sample_content]), call([sample_content]), call([sample_content]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_skipped_content_add_failure( | def test_retrying_proxy_swh_storage_skipped_content_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | "swh.storage.in_memory.InMemoryStorage.skipped_content_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add content_metadata!" | "Refuse to add content_metadata!" | ||||
| ) | ) | ||||
| sample_content = sample_data["skipped_content"][0] | sample_content = sample_data.skipped_content | ||||
| sample_content_dict = sample_content.to_dict() | sample_content_dict = sample_content.to_dict() | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.skipped_content_add([sample_content]) | swh_storage.skipped_content_add([sample_content]) | ||||
| skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | skipped_contents = list(swh_storage.skipped_content_missing([sample_content_dict])) | ||||
| assert len(skipped_contents) == 1 | assert len(skipped_contents) == 1 | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | ||||
| """Standard origin_visit_add works as before | """Standard origin_visit_add works as before | ||||
| """ | """ | ||||
| origin = sample_data["origin"][0] | origin = sample_data.origin | ||||
| visit = sample_data.origin_visit | |||||
| assert visit.origin == origin.url | |||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| origins = list(swh_storage.origin_visit_get(origin.url)) | origins = list(swh_storage.origin_visit_get(origin.url)) | ||||
| assert not origins | assert not origins | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="hg") | |||||
| origin_visit = swh_storage.origin_visit_add([visit])[0] | origin_visit = swh_storage.origin_visit_add([visit])[0] | ||||
| assert origin_visit.origin == origin.url | assert origin_visit.origin == origin.url | ||||
| assert isinstance(origin_visit.visit, int) | assert isinstance(origin_visit.visit, int) | ||||
| origin_visit = next(swh_storage.origin_visit_get(origin.url)) | origin_visit = next(swh_storage.origin_visit_get(origin.url)) | ||||
| assert origin_visit["origin"] == origin.url | assert origin_visit["origin"] == origin.url | ||||
| assert isinstance(origin_visit["visit"], int) | assert isinstance(origin_visit["visit"], int) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_retry( | def test_retrying_proxy_swh_storage_origin_visit_add_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| origin = sample_data["origin"][1] | origin = sample_data.origin | ||||
| visit = sample_data.origin_visit | |||||
| assert visit.origin == origin.url | |||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="git") | |||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("origin already inserted"), | psycopg2.IntegrityError("origin already inserted"), | ||||
| # ok then! | # ok then! | ||||
| [visit], | [visit], | ||||
| ] | ] | ||||
| Show All 13 Lines | def test_retrying_proxy_swh_storage_origin_visit_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | ||||
| origin = sample_data["origin"][0] | origin = sample_data.origin | ||||
| visit = sample_data.origin_visit | |||||
| assert visit.origin == origin.url | |||||
| origins = list(swh_storage.origin_visit_get(origin.url)) | origins = list(swh_storage.origin_visit_get(origin.url)) | ||||
| assert not origins | assert not origins | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| visit = OriginVisit(origin=origin.url, date=date_visit1, type="svn",) | |||||
| swh_storage.origin_visit_add([visit]) | swh_storage.origin_visit_add([visit]) | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([visit]),] | [call([visit]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | ||||
| """Standard metadata_fetcher_add works as before | """Standard metadata_fetcher_add works as before | ||||
| """ | """ | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data.metadata_fetcher | ||||
| metadata_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | metadata_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert not metadata_fetcher | assert not metadata_fetcher | ||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert actual_fetcher == fetcher | assert actual_fetcher == fetcher | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | def test_retrying_proxy_storage_metadata_fetcher_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data.metadata_fetcher | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("metadata_fetcher already inserted"), | psycopg2.IntegrityError("metadata_fetcher already inserted"), | ||||
| Show All 19 Lines | ): | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | "swh.storage.in_memory.InMemoryStorage.metadata_fetcher_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add metadata_fetcher always!" | "Refuse to add metadata_fetcher always!" | ||||
| ) | ) | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data.metadata_fetcher | ||||
| actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) | ||||
| assert not actual_fetcher | assert not actual_fetcher | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.metadata_fetcher_add([fetcher]) | swh_storage.metadata_fetcher_add([fetcher]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data): | def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data): | ||||
| """Standard metadata_authority_add works as before | """Standard metadata_authority_add works as before | ||||
| """ | """ | ||||
| authority = sample_data["authority"][0] | authority = sample_data.metadata_authority | ||||
| assert not swh_storage.metadata_authority_get(authority.type, authority.url) | assert not swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| actual_authority = swh_storage.metadata_authority_get(authority.type, authority.url) | actual_authority = swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| assert actual_authority == authority | assert actual_authority == authority | ||||
| def test_retrying_proxy_storage_metadata_authority_add_with_retry( | def test_retrying_proxy_storage_metadata_authority_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| authority = sample_data["authority"][0] | authority = sample_data.metadata_authority | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| Show All 19 Lines | ): | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | "swh.storage.in_memory.InMemoryStorage.metadata_authority_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add authority_id always!" | "Refuse to add authority_id always!" | ||||
| ) | ) | ||||
| authority = sample_data["authority"][0] | authority = sample_data.metadata_authority | ||||
| swh_storage.metadata_authority_get(authority.type, authority.url) | swh_storage.metadata_authority_get(authority.type, authority.url) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.metadata_authority_add([authority]) | swh_storage.metadata_authority_add([authority]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_object_metadata_add(swh_storage, sample_data): | def test_retrying_proxy_storage_object_metadata_add(swh_storage, sample_data): | ||||
| """Standard object_metadata_add works as before | """Standard object_metadata_add works as before | ||||
| """ | """ | ||||
| origin = sample_data["origin"][0] | origin = sample_data.origin | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data.origin_metadata1 | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| swh_storage.metadata_authority_add([sample_data["authority"][0]]) | swh_storage.metadata_authority_add([sample_data.metadata_authority]) | ||||
| swh_storage.metadata_fetcher_add([sample_data["fetcher"][0]]) | swh_storage.metadata_fetcher_add([sample_data.metadata_fetcher]) | ||||
| origin_metadata = swh_storage.object_metadata_get( | origin_metadata = swh_storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata["next_page_token"] is None | assert origin_metadata["next_page_token"] is None | ||||
| assert not origin_metadata["results"] | assert not origin_metadata["results"] | ||||
| swh_storage.object_metadata_add([ori_meta]) | swh_storage.object_metadata_add([ori_meta]) | ||||
| origin_metadata = swh_storage.object_metadata_get( | origin_metadata = swh_storage.object_metadata_get( | ||||
| MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | MetadataTargetType.ORIGIN, ori_meta.id, ori_meta.authority | ||||
| ) | ) | ||||
| assert origin_metadata | assert origin_metadata | ||||
| def test_retrying_proxy_storage_object_metadata_add_with_retry( | def test_retrying_proxy_storage_object_metadata_add_with_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision, | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| origin = sample_data["origin"][0] | origin = sample_data.origin | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data.origin_metadata1 | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| swh_storage.metadata_authority_add([sample_data["authority"][0]]) | swh_storage.metadata_authority_add([sample_data.metadata_authority]) | ||||
| swh_storage.metadata_fetcher_add([sample_data["fetcher"][0]]) | swh_storage.metadata_fetcher_add([sample_data.metadata_fetcher]) | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| Show All 20 Lines | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch( | mock_memory = mocker.patch( | ||||
| "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | "swh.storage.in_memory.InMemoryStorage.object_metadata_add" | ||||
| ) | ) | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add always!") | ||||
| origin = sample_data["origin"][0] | origin = sample_data.origin | ||||
| ori_meta = sample_data["origin_metadata"][0] | ori_meta = sample_data.origin_metadata1 | ||||
| assert origin.url == ori_meta.id | assert origin.url == ori_meta.id | ||||
| swh_storage.origin_add([origin]) | swh_storage.origin_add([origin]) | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.object_metadata_add([ori_meta]) | swh_storage.object_metadata_add([ori_meta]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | def test_retrying_proxy_storage_directory_add(swh_storage, sample_data): | ||||
| """Standard directory_add works as before | """Standard directory_add works as before | ||||
| """ | """ | ||||
| sample_dir = sample_data["directory"][0] | sample_dir = sample_data.directory | ||||
| directory = swh_storage.directory_get_random() # no directory | directory = swh_storage.directory_get_random() # no directory | ||||
| assert not directory | assert not directory | ||||
| s = swh_storage.directory_add([sample_dir]) | s = swh_storage.directory_add([sample_dir]) | ||||
| assert s == { | assert s == { | ||||
| "directory:add": 1, | "directory:add": 1, | ||||
| } | } | ||||
| Show All 13 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("directory already inserted"), | psycopg2.IntegrityError("directory already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"directory:add": 1}, | {"directory:add": 1}, | ||||
| ] | ] | ||||
| sample_dir = sample_data["directory"][1] | sample_dir = sample_data.directories[1] | ||||
| directory_id = swh_storage.directory_get_random() # no directory | directory_id = swh_storage.directory_get_random() # no directory | ||||
| assert not directory_id | assert not directory_id | ||||
| s = swh_storage.directory_add([sample_dir]) | s = swh_storage.directory_add([sample_dir]) | ||||
| assert s == { | assert s == { | ||||
| "directory:add": 1, | "directory:add": 1, | ||||
| } | } | ||||
| Show All 9 Lines | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.directory_add") | ||||
| mock_memory.side_effect = StorageArgumentException( | mock_memory.side_effect = StorageArgumentException( | ||||
| "Refuse to add directory always!" | "Refuse to add directory always!" | ||||
| ) | ) | ||||
| sample_dir = sample_data["directory"][0] | sample_dir = sample_data.directory | ||||
| directory_id = swh_storage.directory_get_random() # no directory | directory_id = swh_storage.directory_get_random() # no directory | ||||
| assert not directory_id | assert not directory_id | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.directory_add([sample_dir]) | swh_storage.directory_add([sample_dir]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_revision_add(swh_storage, sample_data): | def test_retrying_proxy_storage_revision_add(swh_storage, sample_data): | ||||
| """Standard revision_add works as before | """Standard revision_add works as before | ||||
| """ | """ | ||||
| sample_rev = sample_data["revision"][0] | sample_rev = sample_data.revision | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| s = swh_storage.revision_add([sample_rev]) | s = swh_storage.revision_add([sample_rev]) | ||||
| assert s == { | assert s == { | ||||
| "revision:add": 1, | "revision:add": 1, | ||||
| } | } | ||||
| Show All 13 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("revision already inserted"), | psycopg2.IntegrityError("revision already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"revision:add": 1}, | {"revision:add": 1}, | ||||
| ] | ] | ||||
| sample_rev = sample_data["revision"][0] | sample_rev = sample_data.revision | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| s = swh_storage.revision_add([sample_rev]) | s = swh_storage.revision_add([sample_rev]) | ||||
| assert s == { | assert s == { | ||||
| "revision:add": 1, | "revision:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_rev]), call([sample_rev]), call([sample_rev]),] | [call([sample_rev]), call([sample_rev]), call([sample_rev]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_revision_add_failure( | def test_retrying_proxy_swh_storage_revision_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.revision_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add revision always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add revision always!") | ||||
| sample_rev = sample_data["revision"][0] | sample_rev = sample_data.revision | ||||
| revision = next(swh_storage.revision_get([sample_rev.id])) | revision = next(swh_storage.revision_get([sample_rev.id])) | ||||
| assert not revision | assert not revision | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.revision_add([sample_rev]) | swh_storage.revision_add([sample_rev]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_release_add(swh_storage, sample_data): | def test_retrying_proxy_storage_release_add(swh_storage, sample_data): | ||||
| """Standard release_add works as before | """Standard release_add works as before | ||||
| """ | """ | ||||
| sample_rel = sample_data["release"][0] | sample_rel = sample_data.release | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| s = swh_storage.release_add([sample_rel]) | s = swh_storage.release_add([sample_rel]) | ||||
| assert s == { | assert s == { | ||||
| "release:add": 1, | "release:add": 1, | ||||
| } | } | ||||
| Show All 13 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("release already inserted"), | psycopg2.IntegrityError("release already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"release:add": 1}, | {"release:add": 1}, | ||||
| ] | ] | ||||
| sample_rel = sample_data["release"][0] | sample_rel = sample_data.release | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| s = swh_storage.release_add([sample_rel]) | s = swh_storage.release_add([sample_rel]) | ||||
| assert s == { | assert s == { | ||||
| "release:add": 1, | "release:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_rel]), call([sample_rel]), call([sample_rel]),] | [call([sample_rel]), call([sample_rel]), call([sample_rel]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_release_add_failure( | def test_retrying_proxy_swh_storage_release_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.release_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add release always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add release always!") | ||||
| sample_rel = sample_data["release"][0] | sample_rel = sample_data.release | ||||
| release = next(swh_storage.release_get([sample_rel.id])) | release = next(swh_storage.release_get([sample_rel.id])) | ||||
| assert not release | assert not release | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.release_add([sample_rel]) | swh_storage.release_add([sample_rel]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||
| def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data): | def test_retrying_proxy_storage_snapshot_add(swh_storage, sample_data): | ||||
| """Standard snapshot_add works as before | """Standard snapshot_add works as before | ||||
| """ | """ | ||||
| sample_snap = sample_data["snapshot"][0] | sample_snap = sample_data.snapshot | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| s = swh_storage.snapshot_add([sample_snap]) | s = swh_storage.snapshot_add([sample_snap]) | ||||
| assert s == { | assert s == { | ||||
| "snapshot:add": 1, | "snapshot:add": 1, | ||||
| } | } | ||||
| Show All 13 Lines | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("snapshot already inserted"), | psycopg2.IntegrityError("snapshot already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"snapshot:add": 1}, | {"snapshot:add": 1}, | ||||
| ] | ] | ||||
| sample_snap = sample_data["snapshot"][0] | sample_snap = sample_data.snapshot | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| s = swh_storage.snapshot_add([sample_snap]) | s = swh_storage.snapshot_add([sample_snap]) | ||||
| assert s == { | assert s == { | ||||
| "snapshot:add": 1, | "snapshot:add": 1, | ||||
| } | } | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call([sample_snap]), call([sample_snap]), call([sample_snap]),] | [call([sample_snap]), call([sample_snap]), call([sample_snap]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_snapshot_add_failure( | def test_retrying_proxy_swh_storage_snapshot_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.snapshot_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add snapshot always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add snapshot always!") | ||||
| sample_snap = sample_data["snapshot"][0] | sample_snap = sample_data.snapshot | ||||
| snapshot = swh_storage.snapshot_get(sample_snap.id) | snapshot = swh_storage.snapshot_get(sample_snap.id) | ||||
| assert not snapshot | assert not snapshot | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.snapshot_add([sample_snap]) | swh_storage.snapshot_add([sample_snap]) | ||||
| assert mock_memory.call_count == 1 | assert mock_memory.call_count == 1 | ||||