diff --git a/swh/storage/proxies/tenacious.py b/swh/storage/proxies/tenacious.py --- a/swh/storage/proxies/tenacious.py +++ b/swh/storage/proxies/tenacious.py @@ -76,16 +76,17 @@ """ - tenacious_methods = ( - "content_add", - "skipped_content_add", - "directory_add", - "revision_add", - "extid_add", - "release_add", - "snapshot_add", - "origin_add", - ) + tenacious_methods: Dict[str, str] = { + "content_add": "content", + "content_add_metadata": "content", + "skipped_content_add": "skipped_content", + "directory_add": "directory", + "revision_add": "revision", + "extid_add": "extid", + "release_add": "release", + "snapshot_add": "snapshot", + "origin_add": "origin", + } def __init__( self, @@ -114,7 +115,7 @@ """ add_function = getattr(self.storage, func_name) - object_type = func_name[:-4] # remove the _add suffix + object_type = self.tenacious_methods[func_name] # list of lists of objects; note this to_add list is consumed from the tail to_add: List[List[BaseModel]] = [list(objects)] diff --git a/swh/storage/tests/test_tenacious.py b/swh/storage/tests/test_tenacious.py --- a/swh/storage/tests/test_tenacious.py +++ b/swh/storage/tests/test_tenacious.py @@ -20,6 +20,7 @@ TestStorageGeneratedData as _TestStorageGeneratedData, ) from swh.storage.tests.storage_tests import TestStorage as _TestStorage # noqa +from swh.storage.utils import now data = StorageData() collections = { @@ -69,6 +70,10 @@ def test_content_add_collision(self, swh_storage, sample_data): pass + @pytest.mark.skip(reason="No collision with the tenacious storage") + def test_content_add_metadata_collision(self, swh_storage, sample_data): + pass + @pytest.mark.skip("content_update is not implemented") def test_content_update(self): pass @@ -127,13 +132,23 @@ testdata = [ pytest.param( "content", + "content_add", list(TEST_OBJECTS["content"]), attr.evolve(model.Content.from_data(data=b"too big"), length=1000), attr.evolve(model.Content.from_data(data=b"to fail"), length=1000), id="content", ), + pytest.param( + "content", + "content_add_metadata", + [attr.evolve(cnt, ctime=now()) for cnt in TEST_OBJECTS["content"]], + attr.evolve(model.Content.from_data(data=b"too big"), length=1000, ctime=now()), + attr.evolve(model.Content.from_data(data=b"to fail"), length=1000, ctime=now()), + id="content_metadata", + ), pytest.param( "skipped_content", + "skipped_content_add", list(TEST_OBJECTS["skipped_content"]), attr.evolve( model.SkippedContent.from_data(data=b"too big", reason="too big"), @@ -147,6 +162,7 @@ ), pytest.param( "directory", + "directory_add", list(TEST_OBJECTS["directory"]), data.directory, data.directory2, @@ -154,6 +170,7 @@ ), pytest.param( "revision", + "revision_add", list(TEST_OBJECTS["revision"]), data.revision, data.revision2, @@ -161,6 +178,7 @@ ), pytest.param( "release", + "release_add", list(TEST_OBJECTS["release"]), data.release, data.release2, @@ -168,20 +186,26 @@ ), pytest.param( "snapshot", + "snapshot_add", list(TEST_OBJECTS["snapshot"]), data.snapshot, data.complete_snapshot, id="snapshot", ), pytest.param( - "origin", list(TEST_OBJECTS["origin"]), data.origin, data.origin2, id="origin", + "origin", + "origin_add", + list(TEST_OBJECTS["origin"]), + data.origin, + data.origin2, + id="origin", ), ] class LimitedInMemoryStorage(InMemoryStorage): # forbidden are 'bad1' and 'bad2' arguments of `testdata` - forbidden = [x[0][2] for x in testdata] + [x[0][3] for x in testdata] + forbidden = [x[0][3] for x in testdata] + [x[0][4] for x in testdata] def __init__(self, *args, **kw): self.add_calls = Counter() @@ -194,6 +218,9 @@ def content_add(self, contents): return self._maybe_add(super().content_add, "content", contents) + def content_add_metadata(self, contents): + return self._maybe_add(super().content_add_metadata, "content", contents) + def skipped_content_add(self, skipped_contents): return self._maybe_add( super().skipped_content_add, "skipped_content", skipped_contents @@ -225,8 +252,8 @@ @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) -@pytest.mark.parametrize("object_type, objects, bad1, bad2", testdata) -def test_tenacious_proxy_storage(object_type, objects, bad1, bad2): +@pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) +def test_tenacious_proxy_storage(object_type, add_func_name, objects, bad1, bad2): storage = get_tenacious_storage() tenacious = storage.storage in_memory = tenacious.storage @@ -235,7 +262,7 @@ size = len(objects) - add_func = getattr(storage, f"{object_type}_add") + add_func = getattr(storage, add_func_name) # Note: when checking the LimitedInMemoryStorage.add_calls counter, it's # hard to guess the exact number of calls in the end (depends on the size @@ -333,8 +360,10 @@ @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) -@pytest.mark.parametrize("object_type, objects, bad1, bad2", testdata) -def test_tenacious_proxy_storage_rate_limit(object_type, objects, bad1, bad2): +@pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) +def test_tenacious_proxy_storage_rate_limit( + object_type, add_func_name, objects, bad1, bad2 +): storage = get_tenacious_storage(error_rate_limit={"errors": 1, "window_size": 3}) tenacious = storage.storage in_memory = tenacious.storage @@ -343,7 +372,7 @@ size = len(objects) - add_func = getattr(storage, f"{object_type}_add") + add_func = getattr(storage, add_func_name) # with no insertion failure, no impact s = add_func(objects)