object_type = 'directory', add_func_name = 'directory_add' objects = [Directory(entries=(), id=hash_to_bytes('4b825dc642cb6eb9a060e54bf8d69288fbee4904'), raw_manifest=None), Directory(ent...est=b'tree 34\x0000644 file1.ext\x00\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11')] bad1 = Directory(entries=(DirectoryEntry(name=b'foo', type='file', target=hash_to_bytes('d81cc0710eb6cf9efd5b920a8453e1e07157...4bf8d69288fbee4904'), perms=0o40000)), id=hash_to_bytes('5256e856a0a0898966d6ba14feb4388b8b82d302'), raw_manifest=None) bad2 = Directory(entries=(DirectoryEntry(name=b'oof', type='file', target=hash_to_bytes('36fade77193cb6d2bd826161a0979d64c28ab4fa'), perms=0o100644),), id=hash_to_bytes('8505808532953da7d2581741f01b29c04b1cb9ab'), raw_manifest=None) @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) @pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) def test_tenacious_proxy_storage_rate_limit( object_type, add_func_name, objects, bad1, bad2 ): storage = get_tenacious_storage(error_rate_limit={"errors": 1, "window_size": 3}) tenacious = storage.storage in_memory = tenacious.storage assert isinstance(tenacious, TenaciousProxyStorage) assert isinstance(in_memory, LimitedInMemoryStorage) size = len(objects) add_func = getattr(storage, add_func_name) # with no insertion failure, no impact s = add_func(objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 0 in_memory.reset() tenacious.reset() # with one insertion failure, no impact s = add_func([bad1] + objects) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() s = add_func(objects[: size // 2] + [bad1] + objects[size // 2 :]) assert s.get(f"{object_type}:add", 0) == size assert s.get(f"{object_type}:add:errors", 0) == 1 in_memory.reset() tenacious.reset() # with two consecutive insertion failures, exception is raised with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func([bad1, bad2] + objects) in_memory.reset() tenacious.reset() if size > 2: # with two consecutive insertion failures, exception is raised # (errors not at the beginning) with pytest.raises(RuntimeError, match="Too many insertion errors"): add_func(objects[: size // 2] + [bad1, bad2] + objects[size // 2 :]) in_memory.reset() tenacious.reset() # with two non-consecutive insertion failures, no impact # (errors are far enough to not reach the rate limit) s = add_func( objects[: size // 3] + [bad1] + objects[size // 3 : 2 * (size // 3)] + [bad2] > + objects[2 * (size // 3) :] ) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_tenacious.py:418: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/validate.py:59: in directory_add return self.storage.directory_add(directories) _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <swh.storage.proxies.tenacious.TenaciousProxyStorage object at 0x7f29e0e2f320> func_name = 'directory_add' objects = [Directory(entries=(), id=hash_to_bytes('4b825dc642cb6eb9a060e54bf8d69288fbee4904'), raw_manifest=None), Directory(ent...est=b'tree 34\x0000644 file1.ext\x00\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11\x11')] def _tenacious_add(self, func_name, objects: Iterable[BaseModel]) -> Dict[str, int]: """Enqueue objects to write to the storage. This checks if the queue's threshold is hit. If it is actually write those to the storage. """ add_function = getattr(self.storage, func_name) object_type = self.tenacious_methods[func_name] # list of lists of objects; note this to_add list is consumed from the tail to_add: List[List[BaseModel]] = [list(objects)] n_objs: int = len(to_add[0]) results: CounterT[str] = Counter() retries: int = self._single_object_retries while to_add: if self.rate_queue.limit_reached(): logging.error( "Too many insertion errors have been detected; " "disabling insertions" ) raise RuntimeError( > "Too many insertion errors have been detected; " "disabling insertions" ) E RuntimeError: Too many insertion errors have been detected; disabling insertions .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/tenacious.py:134: RuntimeError