Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/tests/test_replay.py
Show First 20 Lines • Show All 70 Lines • ▼ Show 20 Lines | def test_storage_replayer(replayer_storage_and_client, caplog): | ||||
nb_sent = 0 | nb_sent = 0 | ||||
for object_type, objects in TEST_OBJECTS.items(): | for object_type, objects in TEST_OBJECTS.items(): | ||||
if object_type == "origin_visit": | if object_type == "origin_visit": | ||||
# src.origin_visit_upsert(objects) | # src.origin_visit_upsert(objects) | ||||
for visit in objects: | for visit in objects: | ||||
src.origin_visit_add( | src.origin_visit_add( | ||||
origin_url=visit.origin, date=visit.date, type=visit.type | origin_url=visit.origin, date=visit.date, type=visit.type | ||||
) | ) | ||||
nb_sent += 1 # this adds origin-visit-status as well | |||||
else: | else: | ||||
method = getattr(src, object_type + "_add") | method = getattr(src, object_type + "_add") | ||||
method(objects) | method(objects) | ||||
nb_sent += len(objects) | nb_sent += len(objects) | ||||
caplog.set_level(logging.ERROR, "swh.journal.replay") | caplog.set_level(logging.ERROR, "swh.journal.replay") | ||||
# Fill the destination storage from Kafka | # Fill the destination storage from Kafka | ||||
Show All 27 Lines | def test_storage_play_with_collision(replayer_storage_and_client, caplog): | ||||
# Fill Kafka using a source storage | # Fill Kafka using a source storage | ||||
nb_sent = 0 | nb_sent = 0 | ||||
for object_type, objects in TEST_OBJECTS.items(): | for object_type, objects in TEST_OBJECTS.items(): | ||||
if object_type == "origin_visit": | if object_type == "origin_visit": | ||||
for visit in objects: | for visit in objects: | ||||
src.origin_visit_add( | src.origin_visit_add( | ||||
origin_url=visit.origin, date=visit.date, type=visit.type | origin_url=visit.origin, date=visit.date, type=visit.type | ||||
) | ) | ||||
nb_sent += 1 # this adds origin-visit-status as well | |||||
else: | else: | ||||
method = getattr(src, object_type + "_add") | method = getattr(src, object_type + "_add") | ||||
method(objects) | method(objects) | ||||
nb_sent += len(objects) | nb_sent += len(objects) | ||||
# Create collision in input data | # Create collision in input data | ||||
# These should not be written in the destination | # These should not be written in the destination | ||||
producer = src.journal_writer.journal.producer | producer = src.journal_writer.journal.producer | ||||
▲ Show 20 Lines • Show All 239 Lines • Show Last 20 Lines |