Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_kafka_writer.py
| Show First 20 Lines • Show All 53 Lines • ▼ Show 20 Lines | for obj_type, objs in TEST_OBJECTS.items(): | ||||
| "origin_visit", | "origin_visit", | ||||
| "origin_visit_status", | "origin_visit_status", | ||||
| "raw_extrinsic_metadata", | "raw_extrinsic_metadata", | ||||
| ): | ): | ||||
| method(objs) | method(objs) | ||||
| expected_messages += len(objs) | expected_messages += len(objs) | ||||
| else: | else: | ||||
| assert False, obj_type | assert False, obj_type | ||||
| assert storage.journal_writer is not None | storage.journal_writer.journal.flush() # type: ignore[attr-defined] | ||||
| storage.journal_writer.journal.flush() | |||||
| existing_topics = set( | existing_topics = set( | ||||
| topic | topic | ||||
| for topic in consumer.list_topics(timeout=10).topics.keys() | for topic in consumer.list_topics(timeout=10).topics.keys() | ||||
| if topic.startswith(f"{kafka_prefix}.") # final . to exclude privileged topics | if topic.startswith(f"{kafka_prefix}.") # final . to exclude privileged topics | ||||
| ) | ) | ||||
| assert existing_topics == { | assert existing_topics == { | ||||
| f"{kafka_prefix}.{obj_type}" | f"{kafka_prefix}.{obj_type}" | ||||
| ▲ Show 20 Lines • Show All 44 Lines • ▼ Show 20 Lines | ): | ||||
| for obj_type, objs in TEST_OBJECTS.items(): | for obj_type, objs in TEST_OBJECTS.items(): | ||||
| if obj_type == "origin_visit": | if obj_type == "origin_visit": | ||||
| # these have non-consistent API and are unrelated with what we | # these have non-consistent API and are unrelated with what we | ||||
| # want to test here | # want to test here | ||||
| continue | continue | ||||
| method = getattr(storage, obj_type + "_add") | method = getattr(storage, obj_type + "_add") | ||||
| method(objs) | method(objs) | ||||
| expected_messages += len(objs) | expected_messages += len(objs) | ||||
| assert storage.journal_writer is not None | storage.journal_writer.journal.flush() # type: ignore[attr-defined] | ||||
| storage.journal_writer.journal.flush() | |||||
| existing_topics = set( | existing_topics = set( | ||||
| topic | topic | ||||
| for topic in consumer.list_topics(timeout=10).topics.keys() | for topic in consumer.list_topics(timeout=10).topics.keys() | ||||
| if topic.startswith(kafka_prefix) | if topic.startswith(kafka_prefix) | ||||
| ) | ) | ||||
| assert existing_topics == { | assert existing_topics == { | ||||
| f"{kafka_prefix}.{obj_type}" | f"{kafka_prefix}.{obj_type}" | ||||
| Show All 39 Lines | |||||