Page Menu
Home
Software Heritage
Search
Configure Global Search
Log In
Files
F7163720
D3511.diff
No One
Temporary
Actions
View File
Edit File
Delete File
View Transforms
Subscribe
Mute Notifications
Award Token
Flag For Later
Size
8 KB
Subscribers
None
D3511.diff
View Options
diff --git a/swh/storage/tests/test_retry.py b/swh/storage/tests/test_retry.py
--- a/swh/storage/tests/test_retry.py
+++ b/swh/storage/tests/test_retry.py
@@ -36,12 +36,21 @@
@pytest.fixture
-def swh_storage():
- storage_config = {
+def swh_storage_backend_config():
+ yield {
"cls": "pipeline",
- "steps": [{"cls": "validate"}, {"cls": "retry"}, {"cls": "memory"},],
+ "steps": [{"cls": "retry"}, {"cls": "memory"},],
}
- return get_storage(**storage_config)
+
+
+@pytest.fixture
+def swh_storage(swh_storage_backend_config):
+ return get_storage(**swh_storage_backend_config)
+
+
+@pytest.fixture
+def swh_storage_validate(swh_storage_backend_config):
+ return get_storage(cls="validate", storage=swh_storage_backend_config)
def test_retrying_proxy_storage_content_add(swh_storage, sample_data_model):
@@ -399,27 +408,27 @@
)
-def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage, sample_data):
+def test_retrying_proxy_storage_metadata_fetcher_add(swh_storage_validate, sample_data):
"""Standard metadata_fetcher_add works as before
"""
fetcher = sample_data["fetcher"][0]
- metadata_fetcher = swh_storage.metadata_fetcher_get(
+ metadata_fetcher = swh_storage_validate.metadata_fetcher_get(
fetcher["name"], fetcher["version"]
)
assert not metadata_fetcher
- swh_storage.metadata_fetcher_add(**fetcher)
+ swh_storage_validate.metadata_fetcher_add(**fetcher)
- actual_fetcher = swh_storage.metadata_fetcher_get(
+ actual_fetcher = swh_storage_validate.metadata_fetcher_get(
fetcher["name"], fetcher["version"]
)
assert actual_fetcher == fetcher
def test_retrying_proxy_storage_metadata_fetcher_add_with_retry(
- monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision
+ monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision
):
"""Multiple retries for hash collision and psycopg2 error but finally ok
@@ -437,12 +446,12 @@
[fetcher],
]
- actual_fetcher = swh_storage.metadata_fetcher_get(
+ actual_fetcher = swh_storage_validate.metadata_fetcher_get(
fetcher["name"], fetcher["version"]
)
assert not actual_fetcher
- swh_storage.metadata_fetcher_add(**fetcher)
+ swh_storage_validate.metadata_fetcher_add(**fetcher)
mock_memory.assert_has_calls(
[
@@ -454,7 +463,7 @@
def test_retrying_proxy_swh_storage_metadata_fetcher_add_failure(
- swh_storage, sample_data, mocker
+ swh_storage_validate, sample_data, mocker
):
"""Unfiltered errors are raising without retry
@@ -468,35 +477,39 @@
fetcher = sample_data["fetcher"][0]
- actual_fetcher = swh_storage.metadata_fetcher_get(
+ actual_fetcher = swh_storage_validate.metadata_fetcher_get(
fetcher["name"], fetcher["version"]
)
assert not actual_fetcher
with pytest.raises(StorageArgumentException, match="Refuse to add"):
- swh_storage.metadata_fetcher_add(**fetcher)
+ swh_storage_validate.metadata_fetcher_add(**fetcher)
assert mock_memory.call_count == 1
-def test_retrying_proxy_storage_metadata_authority_add(swh_storage, sample_data):
+def test_retrying_proxy_storage_metadata_authority_add(
+ swh_storage_validate, sample_data
+):
"""Standard metadata_authority_add works as before
"""
authority = sample_data["authority"][0]
- assert not swh_storage.metadata_authority_get(authority["type"], authority["url"])
+ assert not swh_storage_validate.metadata_authority_get(
+ authority["type"], authority["url"]
+ )
- swh_storage.metadata_authority_add(**authority)
+ swh_storage_validate.metadata_authority_add(**authority)
- actual_authority = swh_storage.metadata_authority_get(
+ actual_authority = swh_storage_validate.metadata_authority_get(
authority["type"], authority["url"]
)
assert actual_authority == authority
def test_retrying_proxy_storage_metadata_authority_add_with_retry(
- monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision
+ monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision
):
"""Multiple retries for hash collision and psycopg2 error but finally ok
@@ -515,9 +528,11 @@
None,
]
- assert not swh_storage.metadata_authority_get(authority["type"], authority["url"])
+ assert not swh_storage_validate.metadata_authority_get(
+ authority["type"], authority["url"]
+ )
- swh_storage.metadata_authority_add(**authority)
+ swh_storage_validate.metadata_authority_add(**authority)
authority_arg_names = ("type", "url", "metadata")
authority_args = [authority[key] for key in authority_arg_names]
@@ -527,7 +542,7 @@
def test_retrying_proxy_swh_storage_metadata_authority_add_failure(
- swh_storage, sample_data, mocker
+ swh_storage_validate, sample_data, mocker
):
"""Unfiltered errors are raising without retry
@@ -541,47 +556,47 @@
authority = sample_data["authority"][0]
- swh_storage.metadata_authority_get(authority["type"], authority["url"])
+ swh_storage_validate.metadata_authority_get(authority["type"], authority["url"])
with pytest.raises(StorageArgumentException, match="Refuse to add"):
- swh_storage.metadata_authority_add(**authority)
+ swh_storage_validate.metadata_authority_add(**authority)
assert mock_memory.call_count == 1
-def test_retrying_proxy_storage_origin_metadata_add(swh_storage, sample_data):
+def test_retrying_proxy_storage_origin_metadata_add(swh_storage_validate, sample_data):
"""Standard origin_metadata_add works as before
"""
ori_meta = sample_data["origin_metadata"][0]
- swh_storage.origin_add_one({"url": ori_meta["origin_url"]})
- swh_storage.metadata_authority_add(**sample_data["authority"][0])
- swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0])
+ swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]})
+ swh_storage_validate.metadata_authority_add(**sample_data["authority"][0])
+ swh_storage_validate.metadata_fetcher_add(**sample_data["fetcher"][0])
- origin_metadata = swh_storage.origin_metadata_get(
+ origin_metadata = swh_storage_validate.origin_metadata_get(
ori_meta["origin_url"], ori_meta["authority"]
)
assert origin_metadata["next_page_token"] is None
assert not origin_metadata["results"]
- swh_storage.origin_metadata_add(**ori_meta)
+ swh_storage_validate.origin_metadata_add(**ori_meta)
- origin_metadata = swh_storage.origin_metadata_get(
+ origin_metadata = swh_storage_validate.origin_metadata_get(
ori_meta["origin_url"], ori_meta["authority"]
)
assert origin_metadata
def test_retrying_proxy_storage_origin_metadata_add_with_retry(
- monkeypatch_sleep, swh_storage, sample_data, mocker, fake_hash_collision
+ monkeypatch_sleep, swh_storage_validate, sample_data, mocker, fake_hash_collision
):
"""Multiple retries for hash collision and psycopg2 error but finally ok
"""
ori_meta = sample_data["origin_metadata"][0]
- swh_storage.origin_add_one({"url": ori_meta["origin_url"]})
- swh_storage.metadata_authority_add(**sample_data["authority"][0])
- swh_storage.metadata_fetcher_add(**sample_data["fetcher"][0])
+ swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]})
+ swh_storage_validate.metadata_authority_add(**sample_data["authority"][0])
+ swh_storage_validate.metadata_fetcher_add(**sample_data["fetcher"][0])
mock_memory = mocker.patch(
"swh.storage.in_memory.InMemoryStorage.origin_metadata_add"
)
@@ -596,7 +611,7 @@
]
# No exception raised as insertion finally came through
- swh_storage.origin_metadata_add(**ori_meta)
+ swh_storage_validate.origin_metadata_add(**ori_meta)
mock_memory.assert_has_calls(
[ # 3 calls, as long as error raised
@@ -629,7 +644,7 @@
def test_retrying_proxy_swh_storage_origin_metadata_add_failure(
- swh_storage, sample_data, mocker
+ swh_storage_validate, sample_data, mocker
):
"""Unfiltered errors are raising without retry
@@ -640,10 +655,10 @@
mock_memory.side_effect = StorageArgumentException("Refuse to add always!")
ori_meta = sample_data["origin_metadata"][0]
- swh_storage.origin_add_one({"url": ori_meta["origin_url"]})
+ swh_storage_validate.origin_add_one({"url": ori_meta["origin_url"]})
with pytest.raises(StorageArgumentException, match="Refuse to add"):
- swh_storage.origin_metadata_add(**ori_meta)
+ swh_storage_validate.origin_metadata_add(**ori_meta)
assert mock_memory.call_count == 1
File Metadata
Details
Attached
Mime Type
text/plain
Expires
Thu, Jan 30, 1:59 PM (8 h, 13 m ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217147
Attached To
D3511: test_retry: Drop validate proxy when we can
Event Timeline
Log In to Comment