Page MenuHomeSoftware Heritage
Files F7066323

D5227.id18730.diff
No OneTemporary
Actions

D5227.id18730.diff
View Options

diff --git a/swh/storage/replay.py b/swh/storage/replay.py
--- a/swh/storage/replay.py
+++ b/swh/storage/replay.py
@@ -137,7 +137,22 @@
storage.origin_add(origins)
method = getattr(storage, f"{object_type}_add")
method(model_objs)
- elif object_type in ("directory", "revision", "release", "snapshot", "origin",):
+ elif object_type == "raw_extrinsic_metadata":
+ converted = [RawExtrinsicMetadata.from_dict(o) for o in objects]
+ authorities = {emd.authority for emd in converted}
+ fetchers = {emd.fetcher for emd in converted}
+ storage.metadata_authority_add(authorities)
+ storage.metadata_fetcher_add(fetchers)
+ storage.raw_extrinsic_metadata_add(converted)
+ elif object_type in (
+ "directory",
+ "revision",
+ "release",
+ "snapshot",
+ "origin",
+ "metadata_fetcher",
+ "metadata_authority",
+ ):
method = getattr(storage, object_type + "_add")
method([object_converter_fn[object_type](o) for o in objects])
else:
diff --git a/swh/storage/tests/test_backfill.py b/swh/storage/tests/test_backfill.py
--- a/swh/storage/tests/test_backfill.py
+++ b/swh/storage/tests/test_backfill.py
@@ -4,6 +4,7 @@
# See top-level LICENSE file for more information
import functools
+import logging
from unittest.mock import patch
import pytest
@@ -223,6 +224,7 @@
kafka_prefix: str,
kafka_consumer_group: str,
kafka_server: str,
+ caplog,
):
prefix1 = f"{kafka_prefix}-1"
prefix2 = f"{kafka_prefix}-2"
@@ -256,6 +258,9 @@
for object_type in TEST_OBJECTS:
backfiller.run(object_type, None, None)
+ # Trace log messages for unhandled object types in the replayer
+ caplog.set_level(logging.DEBUG, "swh.storage.replay")
+
# now check journal content are the same under both topics
# use the replayer scaffolding to fill storages to make is a bit easier
# Replaying #1
@@ -282,3 +287,8 @@
# Compare storages
check_replayed(sto1, sto2)
+
+ for record in caplog.records:
+ assert (
+ "this should not happen" not in record.message
+ ), "Replayer ignored some message types, see captured logging"
diff --git a/swh/storage/tests/test_replay.py b/swh/storage/tests/test_replay.py
--- a/swh/storage/tests/test_replay.py
+++ b/swh/storage/tests/test_replay.py
@@ -204,6 +204,7 @@
"origins",
"origin_visits",
"origin_visit_statuses",
+ "raw_extrinsic_metadata",
):
if exclude and attr_ in exclude:
continue
@@ -351,6 +352,7 @@
"snapshots",
"origins",
"origin_visit_statuses",
+ "raw_extrinsic_metadata",
):
expected_objects = [
(id, nullify_ctime(maybe_anonymize(attr_, obj)))

File Metadata

Mime Type
text/plain
Expires
Nov 5 2024, 5:47 AM (8 w, 2 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3217357

Event Timeline

About · Developer information · Legal