object_type = 'content', add_func_name = 'content_add_metadata' objects = [Content(sha1=b'O\xb7kk\x9f\xd9Au\xa3U\x81\xffz\xe5\xe4Ml\xb8\xaa\xbb', sha1_git=b'\x86\xbck7~\x9d%\xf9\xd2gw\xa4\xa2\...='visible', data=b'foo5', ctime=datetime.datetime(2021, 9, 15, 13, 14, 42, 263906, tzinfo=datetime.timezone.utc)), ...] bad1 = Content(sha1=b';@\xceV\xc0_\x9d\xf6k\xc1*E\x92D~o)&\r\xf3', sha1_git=b'\xbdJ\x90,\x07\x0f\xf1)=J#\x98\xe2\xcd\x8a\xf6H...tus='visible', data=b'too big', ctime=datetime.datetime(2021, 9, 15, 13, 14, 42, 264778, tzinfo=datetime.timezone.utc)) bad2 = Content(sha1=b'~W\x86~n\t\x96\xbd\xdc+Y$\xa9\x13c\x054S\xf3%', sha1_git=b'b\xed\x7f\x12\x18)\xaa\xb74\x8e\x0b\xf2\\\xb...tus='visible', data=b'to fail', ctime=datetime.datetime(2021, 9, 15, 13, 14, 42, 264901, tzinfo=datetime.timezone.utc)) @patch("swh.storage.in_memory.InMemoryStorage", LimitedInMemoryStorage) @pytest.mark.parametrize("object_type, add_func_name, objects, bad1, bad2", testdata) def test_tenacious_proxy_storage(object_type, add_func_name, objects, bad1, bad2): storage = get_tenacious_storage() tenacious = storage.storage in_memory = tenacious.storage assert isinstance(tenacious, TenaciousProxyStorage) assert isinstance(in_memory, LimitedInMemoryStorage) size = len(objects) add_func = getattr(storage, add_func_name) # Note: when checking the LimitedInMemoryStorage.add_calls counter, it's # hard to guess the exact number of calls in the end (depends on the size # of batch and the position of bad objects in this batch). So we will only # check a lower limit of the form (n + m), where n is the minimum expected # number of additions (due to the batch begin split), and m is the fact # that bad objects are tried (individually) several (3) times before giving # up. So for one bad object, m is 3; for 2 bad objects, m is 6, etc. > s = add_func(objects) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_tenacious.py:275: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ self = <swh.storage.proxies.tenacious.TenaciousProxyStorage object at 0x7f003b973748> func_name = 'content_add_metadata' objects = [Content(sha1=b'O\xb7kk\x9f\xd9Au\xa3U\x81\xffz\xe5\xe4Ml\xb8\xaa\xbb', sha1_git=b'\x86\xbck7~\x9d%\xf9\xd2gw\xa4\xa2\...='visible', data=b'foo5', ctime=datetime.datetime(2021, 9, 15, 13, 14, 42, 263906, tzinfo=datetime.timezone.utc)), ...] def _tenacious_add(self, func_name, objects: Iterable[BaseModel]) -> Dict[str, int]: """Enqueue objects to write to the storage. This checks if the queue's threshold is hit. If it is actually write those to the storage. """ add_function = getattr(self.storage, func_name) object_type = self.tenacious_methods[func_name] # list of lists of objects; note this to_add list is consumed from the tail to_add: List[List[BaseModel]] = [list(objects)] n_objs: int = len(to_add[0]) results: CounterT[str] = Counter() retries: int = self._single_object_retries while to_add: if self.rate_queue.limit_reached(): logging.error( "Too many insertion errors have been detected; " "disabling insertions" ) raise RuntimeError( > "Too many insertion errors have been detected; " "disabling insertions" ) E RuntimeError: Too many insertion errors have been detected; disabling insertions .tox/py3/lib/python3.7/site-packages/swh/storage/proxies/tenacious.py:134: RuntimeError