Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_retry.py
| Show All 10 Lines | |||||
| from swh.model.model import ( | from swh.model.model import ( | ||||
| Content, | Content, | ||||
| Directory, | Directory, | ||||
| Release, | Release, | ||||
| Revision, | Revision, | ||||
| Snapshot, | Snapshot, | ||||
| SkippedContent, | SkippedContent, | ||||
| Origin, | Origin, | ||||
| OriginVisit, | |||||
| ) | ) | ||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from swh.storage.exc import HashCollision, StorageArgumentException | from swh.storage.exc import HashCollision, StorageArgumentException | ||||
| from .storage_data import date_visit1 | from .storage_data import date_visit1 | ||||
| ▲ Show 20 Lines • Show All 323 Lines • ▼ Show 20 Lines | def test_retrying_proxy_swh_storage_origin_visit_add(swh_storage, sample_data): | ||||
| """ | """ | ||||
| sample_origin = sample_data["origin"][0] | sample_origin = sample_data["origin"][0] | ||||
| origin_url = swh_storage.origin_add_one(sample_origin) | origin_url = swh_storage.origin_add_one(sample_origin) | ||||
| origin = list(swh_storage.origin_visit_get(origin_url)) | origin = list(swh_storage.origin_visit_get(origin_url)) | ||||
| assert not origin | assert not origin | ||||
| origin_visit = swh_storage.origin_visit_add(origin_url, date_visit1, "hg") | visit = OriginVisit( | ||||
| origin=origin_url, date=date_visit1, type="hg", status="ongoing", snapshot=None | |||||
| ) | |||||
| origin_visit = swh_storage.origin_visit_add([visit])[0] | |||||
| assert origin_visit.origin == origin_url | assert origin_visit.origin == origin_url | ||||
| assert isinstance(origin_visit.visit, int) | assert isinstance(origin_visit.visit, int) | ||||
| origin_visit = next(swh_storage.origin_visit_get(origin_url)) | origin_visit = next(swh_storage.origin_visit_get(origin_url)) | ||||
| assert origin_visit["origin"] == origin_url | assert origin_visit["origin"] == origin_url | ||||
| assert isinstance(origin_visit["visit"], int) | assert isinstance(origin_visit["visit"], int) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_retry( | def test_retrying_proxy_swh_storage_origin_visit_add_retry( | ||||
| monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision | ||||
| ): | ): | ||||
| """Multiple retries for hash collision and psycopg2 error but finally ok | """Multiple retries for hash collision and psycopg2 error but finally ok | ||||
| """ | """ | ||||
| sample_origin = sample_data["origin"][1] | sample_origin = sample_data["origin"][1] | ||||
| origin_url = swh_storage.origin_add_one(sample_origin) | origin_url = swh_storage.origin_add_one(sample_origin) | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| visit = OriginVisit( | |||||
| origin=origin_url, date=date_visit1, type="git", status="ongoing", snapshot=None | |||||
| ) | |||||
| mock_memory.side_effect = [ | mock_memory.side_effect = [ | ||||
| # first try goes ko | # first try goes ko | ||||
| fake_hash_collision, | fake_hash_collision, | ||||
| # second try goes ko | # second try goes ko | ||||
| psycopg2.IntegrityError("origin already inserted"), | psycopg2.IntegrityError("origin already inserted"), | ||||
| # ok then! | # ok then! | ||||
| {"origin": origin_url, "visit": 1}, | [visit], | ||||
| ] | ] | ||||
| origin = list(swh_storage.origin_visit_get(origin_url)) | origin = list(swh_storage.origin_visit_get(origin_url)) | ||||
| assert not origin | assert not origin | ||||
| r = swh_storage.origin_visit_add(origin_url, date_visit1, "git") | r = swh_storage.origin_visit_add([visit]) | ||||
| assert r == {"origin": origin_url, "visit": 1} | assert r == [visit] | ||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [ | [call([visit]), call([visit]), call([visit]),] | ||||
| call(origin_url, date_visit1, "git"), | |||||
| call(origin_url, date_visit1, "git"), | |||||
| call(origin_url, date_visit1, "git"), | |||||
| ] | |||||
| ) | ) | ||||
| def test_retrying_proxy_swh_storage_origin_visit_add_failure( | def test_retrying_proxy_swh_storage_origin_visit_add_failure( | ||||
| swh_storage, sample_data, mocker | swh_storage, sample_data, mocker | ||||
| ): | ): | ||||
| """Unfiltered errors are raising without retry | """Unfiltered errors are raising without retry | ||||
| """ | """ | ||||
| mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | mock_memory = mocker.patch("swh.storage.in_memory.InMemoryStorage.origin_visit_add") | ||||
| mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | mock_memory.side_effect = StorageArgumentException("Refuse to add origin always!") | ||||
| origin_url = sample_data["origin"][0]["url"] | origin_url = sample_data["origin"][0]["url"] | ||||
| origin = list(swh_storage.origin_visit_get(origin_url)) | origin = list(swh_storage.origin_visit_get(origin_url)) | ||||
| assert not origin | assert not origin | ||||
| with pytest.raises(StorageArgumentException, match="Refuse to add"): | with pytest.raises(StorageArgumentException, match="Refuse to add"): | ||||
| swh_storage.origin_visit_add(origin_url, date_visit1, "svn") | visit = OriginVisit( | ||||
| origin=origin_url, | |||||
| date=date_visit1, | |||||
| type="svn", | |||||
| status="ongoing", | |||||
| snapshot=None, | |||||
| ) | |||||
| swh_storage.origin_visit_add([visit]) | |||||
| mock_memory.assert_has_calls( | mock_memory.assert_has_calls( | ||||
| [call(origin_url, date_visit1, "svn"),] | [call([visit]),] | ||||
| ) | ) | ||||
| def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data): | ||||
| """Standard metadata_fetcher_add works as before | """Standard metadata_fetcher_add works as before | ||||
| """ | """ | ||||
| fetcher = sample_data["fetcher"][0] | fetcher = sample_data["fetcher"][0] | ||||
| ▲ Show 20 Lines • Show All 242 Lines • ▼ Show 20 Lines | |||||
| def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | def test_retrying_proxy_swh_storage_origin_visit_update(swh_storage, sample_data): | ||||
| """Standard origin_visit_update works as before | """Standard origin_visit_update works as before | ||||
| """ | """ | ||||
| sample_origin = sample_data["origin"][0] | sample_origin = sample_data["origin"][0] | ||||
| origin_url = swh_storage.origin_add_one(sample_origin) | origin_url = swh_storage.origin_add_one(sample_origin) | ||||
| origin_visit = swh_storage.origin_visit_add(origin_url, date_visit1, "hg") | visit = OriginVisit( | ||||
| origin=origin_url, date=date_visit1, type="hg", status="ongoing", snapshot=None | |||||
| ) | |||||
| origin_visit = swh_storage.origin_visit_add([visit])[0] | |||||
| ov = next(swh_storage.origin_visit_get(origin_url)) | ov = next(swh_storage.origin_visit_get(origin_url)) | ||||
| assert ov["origin"] == origin_url | assert ov["origin"] == origin_url | ||||
| assert ov["visit"] == origin_visit.visit | assert ov["visit"] == origin_visit.visit | ||||
| assert ov["status"] == "ongoing" | assert ov["status"] == "ongoing" | ||||
| assert ov["snapshot"] is None | assert ov["snapshot"] is None | ||||
| assert ov["metadata"] is None | assert ov["metadata"] is None | ||||
| ▲ Show 20 Lines • Show All 354 Lines • Show Last 20 Lines | |||||