diff --git a/swh/storage/replay.py b/swh/storage/replay.py --- a/swh/storage/replay.py +++ b/swh/storage/replay.py @@ -137,7 +137,22 @@ storage.origin_add(origins) method = getattr(storage, f"{object_type}_add") method(model_objs) - elif object_type in ("directory", "revision", "release", "snapshot", "origin",): + elif object_type == "raw_extrinsic_metadata": + converted = [RawExtrinsicMetadata.from_dict(o) for o in objects] + authorities = {emd.authority for emd in converted} + fetchers = {emd.fetcher for emd in converted} + storage.metadata_authority_add(authorities) + storage.metadata_fetcher_add(fetchers) + storage.raw_extrinsic_metadata_add(converted) + elif object_type in ( + "directory", + "revision", + "release", + "snapshot", + "origin", + "metadata_fetcher", + "metadata_authority", + ): method = getattr(storage, object_type + "_add") method([object_converter_fn[object_type](o) for o in objects]) else: diff --git a/swh/storage/tests/test_backfill.py b/swh/storage/tests/test_backfill.py --- a/swh/storage/tests/test_backfill.py +++ b/swh/storage/tests/test_backfill.py @@ -4,6 +4,7 @@ # See top-level LICENSE file for more information import functools +import logging from unittest.mock import patch import pytest @@ -223,6 +224,7 @@ kafka_prefix: str, kafka_consumer_group: str, kafka_server: str, + caplog, ): prefix1 = f"{kafka_prefix}-1" prefix2 = f"{kafka_prefix}-2" @@ -256,6 +258,9 @@ for object_type in TEST_OBJECTS: backfiller.run(object_type, None, None) + # Trace log messages for unhandled object types in the replayer + caplog.set_level(logging.DEBUG, "swh.storage.replay") + # now check journal content are the same under both topics # use the replayer scaffolding to fill storages to make is a bit easier # Replaying #1 @@ -282,3 +287,8 @@ # Compare storages check_replayed(sto1, sto2) + + for record in caplog.records: + assert ( + "this should not happen" not in record.message + ), "Replayer ignored some message types, see captured logging" diff --git a/swh/storage/tests/test_replay.py b/swh/storage/tests/test_replay.py --- a/swh/storage/tests/test_replay.py +++ b/swh/storage/tests/test_replay.py @@ -204,6 +204,7 @@ "origins", "origin_visits", "origin_visit_statuses", + "raw_extrinsic_metadata", ): if exclude and attr_ in exclude: continue @@ -351,6 +352,7 @@ "snapshots", "origins", "origin_visit_statuses", + "raw_extrinsic_metadata", ): expected_objects = [ (id, nullify_ctime(maybe_anonymize(attr_, obj)))