Page MenuHomeSoftware Heritage
Paste P690

using datetime.datetime on origin-visit make the test_kafka_writer fail
ActivePublic
Actions

Authored by ardumont on Jun 8 2020, 6:08 PM.
______________________________________________________________________________________________________________ test_kafka_writer ______________________________________________________________________________________________________________
kafka_prefix = 'abcxwtgptr', kafka_server = '127.0.0.1:34923', consumer = <cimpl.Consumer object at 0x7f50d122ea60>, privileged_object_types = {'release', 'revision'}
def test_kafka_writer(
kafka_prefix: str,
kafka_server: str,
consumer: Consumer,
privileged_object_types: Iterable[str],
):
writer = KafkaJournalWriter(
brokers=[kafka_server],
client_id="kafka_writer",
prefix=kafka_prefix,
anonymize=False,
)
expected_messages = 0
for object_type, objects in TEST_OBJECTS.items():
writer.write_additions(object_type, objects)
expected_messages += len(objects)
consumed_messages = consume_messages(consumer, kafka_prefix, expected_messages)
> assert_all_objects_consumed(consumed_messages)
.tox/py3/lib/python3.7/site-packages/swh/journal/tests/test_kafka_writer.py:39:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
consumed_messages = defaultdict(<class 'list'>, {'content': [(b"\xaa\xad\xd9Iw\xb8\xfb\xf3\xf6\xfb\t\xfc;\xbb\xc9\xed\xbd\xfa\x84'", {'sha...\xc20\x00p\xf0V\xe0{", 'target_type': 'snapshot'}}, 'id': b'\xec\xeeH9z\x92\xb0\xd04\xe9u*\x17E\x9f6\x91\xa7>\xf9'})]})
exclude = None
def assert_all_objects_consumed(
consumed_messages: Dict, exclude: Optional[Collection] = None
):
"""Check whether all objects from TEST_OBJECT_DICTS have been consumed
`exclude` can be a list of object types for which we do not want to compare the
values (eg. for anonymized object).
"""
for object_type, known_values in TEST_OBJECT_DICTS.items():
known_keys = [object_key(object_type, obj) for obj in TEST_OBJECTS[object_type]]
if not consumed_messages[object_type]:
return
(received_keys, received_values) = zip(*consumed_messages[object_type])
if object_type in ("content", "skipped_content"):
for value in received_values:
del value["ctime"]
for key in known_keys:
assert key in received_keys, (
f"expected {object_type} key {hash_to_hex(key)} "
"absent from consumed messages"
)
if exclude and object_type in exclude:
continue
for value in known_values:
> assert value in received_values, (
f"expected {object_type} value {value!r} is "
"absent from consumed messages"
)
E AssertionError: expected origin_visit value {'origin': 'https://somewhere.org/den/fox', 'date': datetime.datetime(2013, 5, 7, 4, 20, 39, 369271, tzinfo=datetime.timezone.utc), 'snapshot': None, 'status': 'ongoing', 'metadata': {'foo': 'bar'}, 'type': 'git', 'visit': 1} is absent from consumed messages
E assert {'date': datetime.datetime(2013, 5, 7, 4, 20, 39, 369271, tzinfo=datetime.timezone.utc), 'metadata': {'foo': 'bar'}, 'origin': 'https://somewhere.org/den/fox', 'snapshot': None, ...} in ({'date': '2018-11-27 17:20:39+00:00', 'metadata': {'baz': 'qux'}, 'origin': 'https://somewhere.org/den/fox', 'snapsho...gin': 'https://overtherainbow.org/fox/den', 'snapshot': b'\xec\xeeH9z\x92\xb0\xd04\xe9u*\x17E\x9f6\x91\xa7>\xf9', ...})
About · Developer information · Legal