replayer_storage_and_client = (<swh.storage.in_memory.InMemoryStorage object at 0x7f574f2b6208>, <swh.journal.client.JournalClient object at 0x7f574f2b6240>)
caplog = <_pytest.logging.LogCaptureFixture object at 0x7f574f356c88>
def test_storage_replayer(replayer_storage_and_client, caplog):
"""Optimal replayer scenario.
This:
- writes objects to a source storage
- replayer consumes objects from the topic and replays them
- a destination storage is filled from this
In the end, both storages should have the same content.
"""
src, replayer = replayer_storage_and_client
# Fill Kafka using a source storage
nb_sent = 0
for object_type, objects in TEST_OBJECTS.items():
if object_type == "origin_visit":
# src.origin_visit_upsert(objects)
for visit in objects:
src.origin_visit_add(
> origin_url=visit.origin, date=visit.date, type=visit.type
)
.tox/py3/lib/python3.7/site-packages/swh/storage/tests/test_replay.py:77:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
.tox/py3/lib/python3.7/site-packages/swh/storage/in_memory.py:821: in origin_visit_add
self._origin_visit_status_add_one(visit_update)
.tox/py3/lib/python3.7/site-packages/swh/storage/in_memory.py:831: in _origin_visit_status_add_one
self.journal_writer.origin_visit_status_add([visit_status])
.tox/py3/lib/python3.7/site-packages/swh/storage/writer.py:91: in origin_visit_status_add
self.write_additions("origin_visit_status", visit_statuses)
.tox/py3/lib/python3.7/site-packages/swh/storage/writer.py:48: in write_additions
self.journal.write_additions(obj_type, values)
.tox/py3/lib/python3.7/site-packages/swh/journal/writer/kafka.py:240: in write_additions
self._write_addition(object_type, object_)
.tox/py3/lib/python3.7/site-packages/swh/journal/writer/kafka.py:212: in _write_addition
key = object_key(object_type, object_)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
object_type = 'origin_visit_status'
object_ = OriginVisitStatus(origin='https://somewhere.org/den/fox', visit=1, date=datetime.datetime(2013, 5, 7, 4, 20, 39, 369271, tzinfo=tzlocal()), status='ongoing', snapshot=None, metadata=None)
def object_key(object_type: str, object_) -> KeyType:
if object_type in ("revision", "release", "directory", "snapshot"):
return object_.id
elif object_type == "content":
return object_.sha1 # TODO: use a dict of hashes
elif object_type == "skipped_content":
return {hash: getattr(object_, hash) for hash in DEFAULT_ALGORITHMS}
elif object_type == "origin":
return {"url": object_.url}
elif object_type == "origin_visit":
return {
"origin": object_.origin,
"date": str(object_.date),
}
else:
> raise ValueError("Unknown object type: %s." % object_type)
E ValueError: Unknown object type: origin_visit_status.
.tox/py3/lib/python3.7/site-packages/swh/journal/serializers.py:66: ValueError
TEST RESULT
TEST RESULT
- Run At
- Jun 8 2020, 4:36 PM