object_type = 'release', add_func_name = 'release_add' objects = [Release(name=b'v0.0.1', message=b'foo', target=hash_to_bytes('0404040404040404040404040404040404040404'), target_type...g 102\x00object 0404040404040404040404040404040404040404\ntype commit\ntag v0.0.1\ntagger foo 1234567890 +200\n\nfoo')] bad1 = Release(name=b'v0.0.1', message=b'synthetic release', target=hash_to_bytes('01a7114f36fddd5ef2511b2cadda237a68adbb12')...offset_bytes=b'+0042'), metadata=None, id=hash_to_bytes('f7f222093a18ec60d781070abec4a630c850b837'), raw_manifest=None) bad2 = Release(name=b'v0.0.2', message=b'v0.0.2\nMisc performance improvements + bug fixes', target=hash_to_bytes('a646dd94c9...offset_bytes=b'-0200'), metadata=None, id=hash_to_bytes('db81a26783a3f4a9db07b4759ffc37621f159bb2'), raw_manifest=None) @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) @pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) def test_tenacious_proxy_storage_rate_limit( object_type, add_func_name, objects, bad1, bad2 ): storage = get_tenacious_storage(error_rate_limit={"errors": 1, "window_size": 3}) tenacious = storage.storage in_memory = tenacious.storage assert isinstance(tenacious, TenaciousProxyStorage) assert isinstance(in_memory, LimitedInMemoryStorage) size = len(objects) add_func = getattr(storage, add_func_name) # with no insertion failure, no impact s = add_func(objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 0 in_memory.reset() tenacious.reset() # with one insertion failure, no impact s = add_func([bad1] + objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() s = add_func(objects[: size // 2] + [bad1] + objects[size // 2 :]) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() # with two consecutive insertion failures, exception is raised with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func([bad1, bad2] + objects) in_memory.reset() tenacious.reset() if size > 2: # with two consecutive insertion failures, exception is raised # (errors not at the beginning) with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func(objects[: size // 2] + [bad1, bad2] + objects[size // 2 :]) in_memory.reset() tenacious.reset() # with two non-consecutive insertion failures, no impact # (errors are far enough to not reach the rate limit) s = add_func( objects[: size // 3] + [bad1] + objects[size // 3 : 2 * (size // 3)] + [bad2] > + objects[2 * (size // 3) :] ) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_tenacious.py:418: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/validate.py:67: in release_add return self.storage.release_add(releases) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <swh.storage.proxies.tenacious.TenaciousProxyStorage object at 0x7f27a2969710> func_name = 'release_add' objects = [Release(name=b'v0.0.1', message=b'foo', target=hash_to_bytes('0404040404040404040404040404040404040404'), target_type...g 102\x00object 0404040404040404040404040404040404040404\ntype commit\ntag v0.0.1\ntagger foo 1234567890 +200\n\nfoo')] def _tenacious_add(self, func_name, objects: Iterable[BaseModel]) -> Dict[str, int]: """Enqueue objects to write to the storage. This checks if the queue's threshold is hit. If it is actually write those to the storage. """ add_function = getattr(self.storage, func_name) object_type = self.tenacious_methods[func_name] # list of lists of objects; note this to_add list is consumed from the tail to_add: List[List[BaseModel]] = [list(objects)] n_objs: int = len(to_add[0]) results: CounterT[str] = Counter() retries: int = self._single_object_retries while to_add: if self.rate_queue.limit_reached(): logging.error( "Too many insertion errors have been detected; " "disabling insertions" ) raise RuntimeError( > "Too many insertion errors have been detected; " "disabling insertions" ) E RuntimeError: Too many insertion errors have been detected; disabling insertions .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/tenacious.py:134: RuntimeError