diff --git a/swh/journal/tests/conftest.py b/swh/journal/tests/conftest.py --- a/swh/journal/tests/conftest.py +++ b/swh/journal/tests/conftest.py @@ -141,14 +141,13 @@ }, ] -# From type to tuple (id, ) -OBJECT_TYPE_KEYS = { +TEST_OBJECT_DICTS: Dict[str, Tuple[Optional[str], List[Dict[str, Any]]]] = { "content": ("sha1", CONTENTS), "revision": ("id", REVISIONS), "release": ("id", RELEASES), "origin": (None, ORIGINS), "origin_visit": (None, ORIGIN_VISITS), -} # type: Dict[str, Tuple[Optional[str], List[Dict[str, Any]]]] +} KAFKA_ROOT = os.environ.get("SWH_KAFKA_ROOT") @@ -276,7 +275,7 @@ TEST_CONFIG = { "consumer_id": "swh.journal.consumer", - "object_types": OBJECT_TYPE_KEYS.keys(), + "object_types": TEST_OBJECT_DICTS.keys(), "stop_after_objects": 1, # will read 1 object and stop "storage": {"cls": "memory", "args": {}}, } diff --git a/swh/journal/tests/test_kafka_writer.py b/swh/journal/tests/test_kafka_writer.py --- a/swh/journal/tests/test_kafka_writer.py +++ b/swh/journal/tests/test_kafka_writer.py @@ -18,7 +18,7 @@ from swh.model.model import Content, Origin, BaseModel -from .conftest import OBJECT_TYPE_KEYS +from .conftest import TEST_OBJECT_DICTS MODEL_OBJECTS = {v: k for (k, v) in OBJECT_TYPES.items()} @@ -61,8 +61,8 @@ def assert_all_objects_consumed(consumed_messages): - """Check whether all objects from OBJECT_TYPE_KEYS have been consumed""" - for (object_type, (key_name, objects)) in OBJECT_TYPE_KEYS.items(): + """Check whether all objects from TEST_OBJECT_DICTS have been consumed""" + for (object_type, (key_name, objects)) in TEST_OBJECT_DICTS.items(): (keys, values) = zip(*consumed_messages[object_type]) if key_name: assert list(keys) == [object_[key_name] for object_ in objects] @@ -93,7 +93,7 @@ expected_messages = 0 - for (object_type, (_, objects)) in OBJECT_TYPE_KEYS.items(): + for (object_type, (_, objects)) in TEST_OBJECT_DICTS.items(): for (num, object_d) in enumerate(objects): if object_type == "origin_visit": object_d = {**object_d, "visit": num} @@ -128,7 +128,7 @@ expected_messages = 0 - for (object_type, (_, objects)) in OBJECT_TYPE_KEYS.items(): + for (object_type, (_, objects)) in TEST_OBJECT_DICTS.items(): method = getattr(storage, object_type + "_add") if object_type in ( "content", diff --git a/swh/journal/tests/test_replay.py b/swh/journal/tests/test_replay.py --- a/swh/journal/tests/test_replay.py +++ b/swh/journal/tests/test_replay.py @@ -24,7 +24,7 @@ from swh.model.hashutil import hash_to_hex from swh.model.model import Content -from .conftest import OBJECT_TYPE_KEYS, DUPLICATE_CONTENTS +from .conftest import TEST_OBJECT_DICTS, DUPLICATE_CONTENTS from .utils import MockedJournalClient, MockedKafkaWriter @@ -66,7 +66,7 @@ # Fill Kafka nb_sent = 0 nb_visits = 0 - for (object_type, (_, objects)) in OBJECT_TYPE_KEYS.items(): + for (object_type, (_, objects)) in TEST_OBJECT_DICTS.items(): topic = make_topic(kafka_prefix, object_type) for object_ in objects: key = bytes(random.randint(0, 255) for _ in range(40)) @@ -98,15 +98,17 @@ assert nb_sent == nb_inserted # Check the objects were actually inserted in the storage - assert OBJECT_TYPE_KEYS["revision"][1] == list( - storage.revision_get([rev["id"] for rev in OBJECT_TYPE_KEYS["revision"][1]]) + assert TEST_OBJECT_DICTS["revision"][1] == list( + storage.revision_get([rev["id"] for rev in TEST_OBJECT_DICTS["revision"][1]]) ) - assert OBJECT_TYPE_KEYS["release"][1] == list( - storage.release_get([rel["id"] for rel in OBJECT_TYPE_KEYS["release"][1]]) + assert TEST_OBJECT_DICTS["release"][1] == list( + storage.release_get([rel["id"] for rel in TEST_OBJECT_DICTS["release"][1]]) ) - origins = list(storage.origin_get([orig for orig in OBJECT_TYPE_KEYS["origin"][1]])) - assert OBJECT_TYPE_KEYS["origin"][1] == [{"url": orig["url"]} for orig in origins] + origins = list( + storage.origin_get([orig for orig in TEST_OBJECT_DICTS["origin"][1]]) + ) + assert TEST_OBJECT_DICTS["origin"][1] == [{"url": orig["url"]} for orig in origins] for origin in origins: origin_url = origin["url"] expected_visits = [ @@ -115,7 +117,7 @@ "origin": origin_url, "date": dateutil.parser.parse(visit["date"]), } - for visit in OBJECT_TYPE_KEYS["origin_visit"][1] + for visit in TEST_OBJECT_DICTS["origin_visit"][1] if visit["origin"] == origin["url"] ] actual_visits = list(storage.origin_visit_get(origin_url)) @@ -123,7 +125,7 @@ del visit["visit"] # opaque identifier assert expected_visits == actual_visits - input_contents = OBJECT_TYPE_KEYS["content"][1] + input_contents = TEST_OBJECT_DICTS["content"][1] contents = storage.content_get_metadata([cont["sha1"] for cont in input_contents]) assert len(contents) == len(input_contents) assert contents == {cont["sha1"]: [cont] for cont in input_contents} @@ -169,7 +171,7 @@ # Fill Kafka nb_sent = 0 nb_visits = 0 - for (object_type, (_, objects)) in OBJECT_TYPE_KEYS.items(): + for (object_type, (_, objects)) in TEST_OBJECT_DICTS.items(): topic = make_topic(kafka_prefix, object_type) for object_ in objects: key = bytes(random.randint(0, 255) for _ in range(40)) @@ -211,15 +213,17 @@ assert nb_sent == nb_inserted # Check the objects were actually inserted in the storage - assert OBJECT_TYPE_KEYS["revision"][1] == list( - storage.revision_get([rev["id"] for rev in OBJECT_TYPE_KEYS["revision"][1]]) + assert TEST_OBJECT_DICTS["revision"][1] == list( + storage.revision_get([rev["id"] for rev in TEST_OBJECT_DICTS["revision"][1]]) ) - assert OBJECT_TYPE_KEYS["release"][1] == list( - storage.release_get([rel["id"] for rel in OBJECT_TYPE_KEYS["release"][1]]) + assert TEST_OBJECT_DICTS["release"][1] == list( + storage.release_get([rel["id"] for rel in TEST_OBJECT_DICTS["release"][1]]) ) - origins = list(storage.origin_get([orig for orig in OBJECT_TYPE_KEYS["origin"][1]])) - assert OBJECT_TYPE_KEYS["origin"][1] == [{"url": orig["url"]} for orig in origins] + origins = list( + storage.origin_get([orig for orig in TEST_OBJECT_DICTS["origin"][1]]) + ) + assert TEST_OBJECT_DICTS["origin"][1] == [{"url": orig["url"]} for orig in origins] for origin in origins: origin_url = origin["url"] expected_visits = [ @@ -228,7 +232,7 @@ "origin": origin_url, "date": dateutil.parser.parse(visit["date"]), } - for visit in OBJECT_TYPE_KEYS["origin_visit"][1] + for visit in TEST_OBJECT_DICTS["origin_visit"][1] if visit["origin"] == origin["url"] ] actual_visits = list(storage.origin_visit_get(origin_url)) @@ -236,7 +240,7 @@ del visit["visit"] # opaque identifier assert expected_visits == actual_visits - input_contents = OBJECT_TYPE_KEYS["content"][1] + input_contents = TEST_OBJECT_DICTS["content"][1] contents = storage.content_get_metadata([cont["sha1"] for cont in input_contents]) assert len(contents) == len(input_contents) assert contents == {cont["sha1"]: [cont] for cont in input_contents}