objstorages = {'dst': <swh.objstorage.replayer.tests.test_cli.FlakyObjStorage object at 0x7f90a4746ef0>, 'src': <swh.objstorage.replayer.tests.test_cli.FlakyObjStorage object at 0x7f90a4a18828>} kafka_prefix = 'fsiytuuxii', kafka_consumer_group = 'test-consumer-fsiytuuxii' kafka_server = '127.0.0.1:50245' caplog = <_pytest.logging.LogCaptureFixture object at 0x7f909bf9afd0> monkeypatch_retry_sleep = None redis_proc = <pytest_redis.executor.RedisExecutor: "/usr/bin/r..." 0x7f909bf8a048> redisdb = Redis<ConnectionPool<UnixDomainSocketConnection<path=/tmp/pytest-of-jenkins/pytest-0/pytest-redis-redis_proc0/redis.12951.sock,db=0>>> @_patch_objstorages(["src", "dst"]) def test_replay_content_failed_copy_retry( objstorages, kafka_prefix: str, kafka_consumer_group: str, kafka_server: Tuple[Popen, int], caplog, monkeypatch_retry_sleep, redis_proc, redisdb, ): """Check the content replayer with a flaky src and dst objstorages for 'get' and 'add' operations, and a few non-recoverable failures (some objects failed to be replayed). """ contents = _fill_objstorage_and_kafka( kafka_server, kafka_prefix, objstorages["src"] ) add_failures = {} get_failures = {} definitely_failed = set() # We want to generate operations failing 1 to CONTENT_REPLAY_RETRIES times. # We generate failures for 2 different operations, get and add. num_retry_contents = 2 * CONTENT_REPLAY_RETRIES assert ( num_retry_contents < NUM_CONTENTS ), "Need to generate more test contents to properly test retry behavior" for i, sha1 in enumerate(contents): if i >= num_retry_contents: break # This generates a number of failures, up to CONTENT_REPLAY_RETRIES num_failures = (i % CONTENT_REPLAY_RETRIES) + 1 # This generates failures of add for the first CONTENT_REPLAY_RETRIES # objects, then failures of get. if i < CONTENT_REPLAY_RETRIES: add_failures["add", sha1] = num_failures else: get_failures["get", sha1] = num_failures # Only contents that have CONTENT_REPLAY_RETRIES or more are # definitely failing if num_failures >= CONTENT_REPLAY_RETRIES: definitely_failed.add(hash_to_hex(sha1)) assert add_failures assert get_failures assert definitely_failed objstorages["dst"] = FlakyObjStorage( state=objstorages["dst"].state, failures=add_failures, ) objstorages["src"] = FlakyObjStorage( state=objstorages["src"].state, failures=get_failures, ) caplog.set_level(logging.DEBUG, "swh.objstorage.replayer.replay") result = invoke( "replay", "--stop-after-objects", str(NUM_CONTENTS), journal_config={ "brokers": kafka_server, "group_id": kafka_consumer_group, "prefix": kafka_prefix, "error_reporter": {"host": redis_proc.host, "port": redis_proc.port}, }, ) expected = r"Done.\n" > assert result.exit_code == 0, result.output E AssertionError: E assert 1 == 0 E +1 E -0 .tox/py3/lib/python3.7/site-packages/swh/objstorage/replayer/tests/test_cli.py:555: AssertionError