Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_kafka_writer.py
Show All 37 Lines | def test_storage_direct_writer(kafka_prefix: str, kafka_server, consumer: Consumer): | ||||
expected_messages = 0 | expected_messages = 0 | ||||
for obj_type, objs in TEST_OBJECTS.items(): | for obj_type, objs in TEST_OBJECTS.items(): | ||||
method = getattr(storage, obj_type + "_add") | method = getattr(storage, obj_type + "_add") | ||||
if obj_type in ( | if obj_type in ( | ||||
"content", | "content", | ||||
"skipped_content", | "skipped_content", | ||||
"directory", | "directory", | ||||
"metadata_authority", | |||||
"metadata_fetcher", | |||||
"revision", | "revision", | ||||
"release", | "release", | ||||
"snapshot", | "snapshot", | ||||
"origin", | "origin", | ||||
"origin_visit_status", | "origin_visit_status", | ||||
"raw_extrinsic_metadata", | |||||
): | ): | ||||
method(objs) | method(objs) | ||||
expected_messages += len(objs) | expected_messages += len(objs) | ||||
elif obj_type in ("origin_visit",): | elif obj_type in ("origin_visit",): | ||||
for obj in objs: | for obj in objs: | ||||
assert isinstance(obj, OriginVisit) | assert isinstance(obj, OriginVisit) | ||||
storage.origin_add([Origin(url=obj.origin)]) | storage.origin_add([Origin(url=obj.origin)]) | ||||
method([obj]) | method([obj]) | ||||
expected_messages += 1 + 1 # 1 visit + 1 visit status | expected_messages += 1 + 1 # 1 visit + 1 visit status | ||||
else: | else: | ||||
assert False, obj_type | assert False, obj_type | ||||
existing_topics = set( | existing_topics = set( | ||||
topic | topic | ||||
for topic in consumer.list_topics(timeout=10).topics.keys() | for topic in consumer.list_topics(timeout=10).topics.keys() | ||||
if topic.startswith(f"{kafka_prefix}.") # final . to exclude privileged topics | if topic.startswith(f"{kafka_prefix}.") # final . to exclude privileged topics | ||||
) | ) | ||||
assert existing_topics == { | assert existing_topics == { | ||||
f"{kafka_prefix}.{obj_type}" | f"{kafka_prefix}.{obj_type}" | ||||
for obj_type in ( | for obj_type in ( | ||||
"content", | "content", | ||||
"directory", | "directory", | ||||
"metadata_authority", | |||||
"metadata_fetcher", | |||||
"origin", | "origin", | ||||
"origin_visit", | "origin_visit", | ||||
"origin_visit_status", | "origin_visit_status", | ||||
"raw_extrinsic_metadata", | |||||
"release", | "release", | ||||
"revision", | "revision", | ||||
"snapshot", | "snapshot", | ||||
"skipped_content", | "skipped_content", | ||||
) | ) | ||||
} | } | ||||
consumed_messages = consume_messages(consumer, kafka_prefix, expected_messages) | consumed_messages = consume_messages(consumer, kafka_prefix, expected_messages) | ||||
Show All 34 Lines | existing_topics = set( | ||||
for topic in consumer.list_topics(timeout=10).topics.keys() | for topic in consumer.list_topics(timeout=10).topics.keys() | ||||
if topic.startswith(kafka_prefix) | if topic.startswith(kafka_prefix) | ||||
) | ) | ||||
assert existing_topics == { | assert existing_topics == { | ||||
f"{kafka_prefix}.{obj_type}" | f"{kafka_prefix}.{obj_type}" | ||||
for obj_type in ( | for obj_type in ( | ||||
"content", | "content", | ||||
"directory", | "directory", | ||||
"metadata_authority", | |||||
"metadata_fetcher", | |||||
"origin", | "origin", | ||||
"origin_visit", | "origin_visit", | ||||
"origin_visit_status", | "origin_visit_status", | ||||
"raw_extrinsic_metadata", | |||||
"release", | "release", | ||||
"revision", | "revision", | ||||
"snapshot", | "snapshot", | ||||
"skipped_content", | "skipped_content", | ||||
) | ) | ||||
} | { | } | { | ||||
f"{kafka_prefix}_privileged.{obj_type}" for obj_type in ("release", "revision",) | f"{kafka_prefix}_privileged.{obj_type}" for obj_type in ("release", "revision",) | ||||
} | } | ||||
Show All 17 Lines |