kafka_prefix = 'lzvdwfoflx', kafka_server = '127.0.0.1:37911' consumer = <cimpl.Consumer object at 0x7f913d93f228> def test_storage_direct_writer(kafka_prefix: str, kafka_server, consumer: Consumer): writer_config = { "cls": "kafka", "brokers": [kafka_server], "client_id": "kafka_writer", "prefix": kafka_prefix, "anonymize": False, } storage_config: Dict[str, Any] = { "cls": "pipeline", "steps": [{"cls": "memory", "journal_writer": writer_config},], } storage = get_storage(**storage_config) expected_messages = 0 for obj_type, objs in TEST_OBJECTS.items(): method = getattr(storage, obj_type + "_add") if obj_type in ( "content", "skipped_content", "directory", "extid", "metadata_authority", "metadata_fetcher", "revision", "release", "snapshot", "origin", "origin_visit_status", "raw_extrinsic_metadata", ): method(objs) expected_messages += len(objs) elif obj_type in ("origin_visit",): for obj in objs: assert isinstance(obj, OriginVisit) storage.origin_add([Origin(url=obj.origin)]) method([obj]) expected_messages += 1 + 1 # 1 visit + 1 visit status else: assert False, obj_type existing_topics = set( topic for topic in consumer.list_topics(timeout=10).topics.keys() if topic.startswith(f"{kafka_prefix}.") # final . to exclude privileged topics ) assert existing_topics == { f"{kafka_prefix}.{obj_type}" for obj_type in ( "content", "directory", "extid", "metadata_authority", "metadata_fetcher", "origin", "origin_visit", "origin_visit_status", "raw_extrinsic_metadata", "release", "revision", "snapshot", "skipped_content", ) } > consumed_messages = consume_messages(consumer, kafka_prefix, expected_messages) .tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_kafka_writer.py:89: _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ consumer = <cimpl.Consumer object at 0x7f913d93f228> kafka_prefix = 'lzvdwfoflx', expected_messages = 53 def consume_messages(consumer, kafka_prefix, expected_messages): """Consume expected_messages from the consumer; Sort them all into a consumed_objects dict""" consumed_messages = defaultdict(list) fetched_messages = 0 retries_left = 1000 while fetched_messages < expected_messages: if retries_left == 0: raise ValueError( > "Timed out fetching messages from kafka. " f"Only {fetched_messages}/{expected_messages} fetched" ) E ValueError: Timed out fetching messages from kafka. Only 33/53 fetched .tox/py3/lib/python3.7/site-packages/swh/journal/pytest_plugin.py:50: ValueError