diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -972,6 +972,8 @@ def raw_extrinsic_metadata_add( self, metadata: Iterable[RawExtrinsicMetadata] ) -> None: + metadata = list(metadata) + self.journal_writer.raw_extrinsic_metadata_add(metadata) for metadata_entry in metadata: if not self._cql_runner.metadata_authority_get( metadata_entry.authority.type.value, metadata_entry.authority.url @@ -1108,6 +1110,8 @@ } def metadata_fetcher_add(self, fetchers: Iterable[MetadataFetcher]) -> None: + fetchers = list(fetchers) + self.journal_writer.metadata_fetcher_add(fetchers) for fetcher in fetchers: self._cql_runner.metadata_fetcher_add( fetcher.name, @@ -1129,6 +1133,8 @@ return None def metadata_authority_add(self, authorities: Iterable[MetadataAuthority]) -> None: + authorities = list(authorities) + self.journal_writer.metadata_authority_add(authorities) for authority in authorities: self._cql_runner.metadata_authority_add( authority.url, diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -1021,6 +1021,8 @@ def raw_extrinsic_metadata_add( self, metadata: Iterable[RawExtrinsicMetadata], ) -> None: + metadata = list(metadata) + self.journal_writer.raw_extrinsic_metadata_add(metadata) for metadata_entry in metadata: authority_key = self._metadata_authority_key(metadata_entry.authority) if authority_key not in self._metadata_authorities: @@ -1131,6 +1133,8 @@ } def metadata_fetcher_add(self, fetchers: Iterable[MetadataFetcher]) -> None: + fetchers = list(fetchers) + self.journal_writer.metadata_fetcher_add(fetchers) for fetcher in fetchers: if fetcher.metadata is None: raise StorageArgumentException( @@ -1148,6 +1152,8 @@ ) def metadata_authority_add(self, authorities: Iterable[MetadataAuthority]) -> None: + authorities = list(authorities) + self.journal_writer.metadata_authority_add(authorities) for authority in authorities: if authority.metadata is None: raise StorageArgumentException( diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -1117,6 +1117,8 @@ def raw_extrinsic_metadata_add( self, metadata: Iterable[RawExtrinsicMetadata], db, cur, ) -> None: + metadata = list(metadata) + self.journal_writer.raw_extrinsic_metadata_add(metadata) counter = Counter[MetadataTargetType]() for metadata_entry in metadata: authority_id = self._get_authority_id(metadata_entry.authority, db, cur) @@ -1255,6 +1257,8 @@ def metadata_fetcher_add( self, fetchers: Iterable[MetadataFetcher], db=None, cur=None ) -> None: + fetchers = list(fetchers) + self.journal_writer.metadata_fetcher_add(fetchers) count = 0 for fetcher in fetchers: if fetcher.metadata is None: @@ -1282,6 +1286,8 @@ def metadata_authority_add( self, authorities: Iterable[MetadataAuthority], db=None, cur=None ) -> None: + authorities = list(authorities) + self.journal_writer.metadata_authority_add(authorities) count = 0 for authority in authorities: if authority.metadata is None: diff --git a/swh/storage/tests/test_storage.py b/swh/storage/tests/test_storage.py --- a/swh/storage/tests/test_storage.py +++ b/swh/storage/tests/test_storage.py @@ -3157,6 +3157,14 @@ res = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) assert res == fetcher + actual_objects = list(swh_storage.journal_writer.journal.objects) + expected_objects = [ + ("metadata_fetcher", fetcher), + ] + + for obj in expected_objects: + assert obj in actual_objects + def test_metadata_fetcher_add_zero(self, swh_storage, sample_data): fetcher = sample_data.metadata_fetcher actual_fetcher = swh_storage.metadata_fetcher_get(fetcher.name, fetcher.version) @@ -3177,6 +3185,14 @@ res = swh_storage.metadata_authority_get(authority.type, authority.url) assert res == authority + actual_objects = list(swh_storage.journal_writer.journal.objects) + expected_objects = [ + ("metadata_authority", authority), + ] + + for obj in expected_objects: + assert obj in actual_objects + def test_metadata_authority_add_zero(self, swh_storage, sample_data): authority = sample_data.metadata_authority @@ -3210,6 +3226,15 @@ content_metadata ) + actual_objects = list(swh_storage.journal_writer.journal.objects) + expected_objects = [ + ("metadata_authority", authority), + ("metadata_fetcher", fetcher), + ] + [("raw_extrinsic_metadata", item) for item in content_metadata] + + for obj in expected_objects: + assert obj in actual_objects + def test_content_metadata_add_duplicate(self, swh_storage, sample_data): """Duplicates should be silently updated.""" content = sample_data.content @@ -3438,6 +3463,17 @@ origin_metadata2, ] + actual_objects = list(swh_storage.journal_writer.journal.objects) + expected_objects = [ + ("metadata_authority", authority), + ("metadata_fetcher", fetcher), + ("raw_extrinsic_metadata", origin_metadata), + ("raw_extrinsic_metadata", origin_metadata2), + ] + + for obj in expected_objects: + assert obj in actual_objects + def test_origin_metadata_add_duplicate(self, swh_storage, sample_data): """Duplicates should be silently updated.""" origin = sample_data.origin diff --git a/swh/storage/writer.py b/swh/storage/writer.py --- a/swh/storage/writer.py +++ b/swh/storage/writer.py @@ -17,6 +17,9 @@ Release, Content, SkippedContent, + RawExtrinsicMetadata, + MetadataFetcher, + MetadataAuthority, ) try: @@ -86,3 +89,14 @@ def origin_add(self, origins: Iterable[Origin]) -> None: self.write_additions("origin", origins) + + def raw_extrinsic_metadata_add( + self, metadata: Iterable[RawExtrinsicMetadata] + ) -> None: + self.write_additions("raw_extrinsic_metadata", metadata) + + def metadata_fetcher_add(self, fetchers: Iterable[MetadataFetcher]) -> None: + self.write_additions("metadata_fetcher", fetchers) + + def metadata_authority_add(self, authorities: Iterable[MetadataAuthority]) -> None: + self.write_additions("metadata_authority", authorities)