object_type = 'revision', add_func_name = 'revision_add' objects = [Revision(message=b'hello', author=Person(fullname=b'foo', name=b'foo', email=b''), committer=Person(fullname=b'foo', ...parent 9B918DD063CEC85C2BC63CC7F167E29F5894DCBCnauthor foo 1234567891 +0200\ncommitter foo 1234567891 +0200\n\nhello')] bad1 = Revision(message=b'hello', author=Person(fullname=b'Nicolas Dandrimont <nicolas@example.com> ', name=b'Nicolas Dandrim...ders=((b'gpgsig', b'test123'), (b'mergetag', b'foo\\bar'), (b'mergetag', b'"\xaf\x89\x80\x01\x00')), raw_manifest=None) bad2 = Revision(message=b'hello again', author=Person(fullname=b'Roberto Dicosmo <roberto@example.com>', name=b'Roberto Dicos...a#zh\xad\xbb\x12',), id=hash_to_bytes('a646dd94c912829659b22a1e7e143d2fa5ebde1b'), extra_headers=(), raw_manifest=None) @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) @pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) def test_tenacious_proxy_storage_rate_limit( object_type, add_func_name, objects, bad1, bad2 ): storage = get_tenacious_storage(error_rate_limit={"errors": 1, "window_size": 3}) tenacious = storage.storage in_memory = tenacious.storage assert isinstance(tenacious, TenaciousProxyStorage) assert isinstance(in_memory, LimitedInMemoryStorage) size = len(objects) add_func = getattr(storage, add_func_name) # with no insertion failure, no impact s = add_func(objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 0 in_memory.reset() tenacious.reset() # with one insertion failure, no impact s = add_func([bad1] + objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() s = add_func(objects[: size // 2] + [bad1] + objects[size // 2 :]) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() # with two consecutive insertion failures, exception is raised with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func([bad1, bad2] + objects) in_memory.reset() tenacious.reset() if size > 2: # with two consecutive insertion failures, exception is raised # (errors not at the beginning) with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func(objects[: size // 2] + [bad1, bad2] + objects[size // 2 :]) in_memory.reset() tenacious.reset() # with two non-consecutive insertion failures, no impact # (errors are far enough to not reach the rate limit) s = add_func( objects[: size // 3] + [bad1] + objects[size // 3 : 2 * (size // 3)] + [bad2] > + objects[2 * (size // 3) :] ) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_tenacious.py:418: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/validate.py:63: in revision_add return self.storage.revision_add(revisions) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <swh.storage.proxies.tenacious.TenaciousProxyStorage object at 0x7f27a0616978> func_name = 'revision_add' objects = [Revision(message=b'hello', author=Person(fullname=b'foo', name=b'foo', email=b''), committer=Person(fullname=b'foo', ...parent 9B918DD063CEC85C2BC63CC7F167E29F5894DCBCnauthor foo 1234567891 +0200\ncommitter foo 1234567891 +0200\n\nhello')] def _tenacious_add(self, func_name, objects: Iterable[BaseModel]) -> Dict[str, int]: """Enqueue objects to write to the storage. This checks if the queue's threshold is hit. If it is actually write those to the storage. """ add_function = getattr(self.storage, func_name) object_type = self.tenacious_methods[func_name] # list of lists of objects; note this to_add list is consumed from the tail to_add: List[List[BaseModel]] = [list(objects)] n_objs: int = len(to_add[0]) results: CounterT[str] = Counter() retries: int = self._single_object_retries while to_add: if self.rate_queue.limit_reached(): logging.error( "Too many insertion errors have been detected; " "disabling insertions" ) raise RuntimeError( > "Too many insertion errors have been detected; " "disabling insertions" ) E RuntimeError: Too many insertion errors have been detected; disabling insertions .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/tenacious.py:134: RuntimeError