diff --git a/swh/storage/cassandra/cql.py b/swh/storage/cassandra/cql.py --- a/swh/storage/cassandra/cql.py +++ b/swh/storage/cassandra/cql.py @@ -62,7 +62,6 @@ ExtIDRow, MetadataAuthorityRow, MetadataFetcherRow, - ObjectCountRow, OriginRow, OriginVisitRow, OriginVisitStatusRow, @@ -302,24 +301,13 @@ ) -> ResultSet: return execute_concurrent_with_args(self._session, statement, args_list) - @_prepared_statement( - "UPDATE object_count SET count = count + ? " - "WHERE partition_key = 0 AND object_type = ?" - ) - def _increment_counter( - self, object_type: str, nb: int, *, statement: PreparedStatement - ) -> None: - self._execute_with_retries(statement, [nb, object_type]) - def _add_one(self, statement, obj: BaseRow) -> None: - self._increment_counter(obj.TABLE, 1) self._execute_with_retries(statement, dataclasses.astuple(obj)) def _add_many(self, statement, objs: Sequence[BaseRow]) -> None: tables = {obj.TABLE for obj in objs} assert len(tables) == 1, f"Cannot insert to multiple tables: {tables}" (table,) = tables - self._increment_counter(table, len(objs)) self._execute_many_with_retries(statement, list(map(dataclasses.astuple, objs))) _T = TypeVar("_T", bound=BaseRow) @@ -354,7 +342,6 @@ """Returned currified by content_add_prepare, to be called when the content row should be added to the primary table.""" self._execute_with_retries(statement, None) - self._increment_counter("content", 1) @_prepared_insert_statement(ContentRow) def content_add_prepare( @@ -508,7 +495,6 @@ """Returned currified by skipped_content_add_prepare, to be called when the content row should be added to the primary table.""" self._execute_with_retries(statement, None) - self._increment_counter("skipped_content", 1) @_prepared_insert_statement(SkippedContentRow) def skipped_content_add_prepare( @@ -1295,7 +1281,6 @@ """Returned currified by extid_add_prepare, to be called when the extid row should be added to the primary table.""" self._execute_with_retries(statement, None) - self._increment_counter("extid", 1) @_prepared_insert_statement(ExtIDRow) def extid_add_prepare( @@ -1407,7 +1392,3 @@ @_prepared_statement("SELECT uuid() FROM revision LIMIT 1;") def check_read(self, *, statement): self._execute_with_retries(statement, []) - - @_prepared_select_statement(ObjectCountRow, "WHERE partition_key=0") - def stat_counters(self, *, statement) -> Iterable[ObjectCountRow]: - return map(ObjectCountRow.from_dict, self._execute_with_retries(statement, [])) diff --git a/swh/storage/cassandra/model.py b/swh/storage/cassandra/model.py --- a/swh/storage/cassandra/model.py +++ b/swh/storage/cassandra/model.py @@ -300,17 +300,6 @@ authority_url: str -@dataclasses.dataclass -class ObjectCountRow(BaseRow): - TABLE = "object_count" - PARTITION_KEY = ("partition_key",) - CLUSTERING_KEY = ("object_type",) - - partition_key: int - object_type: str - count: int - - @dataclasses.dataclass class ExtIDRow(BaseRow): TABLE = "extid" diff --git a/swh/storage/cassandra/schema.py b/swh/storage/cassandra/schema.py --- a/swh/storage/cassandra/schema.py +++ b/swh/storage/cassandra/schema.py @@ -267,13 +267,6 @@ PRIMARY KEY ((id)) );""", """ -CREATE TABLE IF NOT EXISTS object_count ( - partition_key smallint, -- Constant, must always be 0 - object_type ascii, - count counter, - PRIMARY KEY ((partition_key), object_type) -);""", - """ CREATE TABLE IF NOT EXISTS extid ( extid_type ascii, extid blob, @@ -319,7 +312,6 @@ "origin_visit", "origin", "raw_extrinsic_metadata", - "object_count", "origin_visit_status", "metadata_authority", "metadata_fetcher", diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -1290,20 +1290,7 @@ return None def stat_counters(self): - rows = self._cql_runner.stat_counters() - keys = ( - "content", - "directory", - "origin", - "origin_visit", - "release", - "revision", - "skipped_content", - "snapshot", - ) - stats = {key: 0 for key in keys} - stats.update({row.object_type: row.count for row in rows}) - return stats + raise NotImplementedError() def refresh_stat_counters(self): pass diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -34,7 +34,6 @@ ExtIDRow, MetadataAuthorityRow, MetadataFetcherRow, - ObjectCountRow, OriginRow, OriginVisitRow, OriginVisitStatusRow, @@ -171,14 +170,6 @@ self._raw_extrinsic_metadata = Table(RawExtrinsicMetadataRow) self._raw_extrinsic_metadata_by_id = Table(RawExtrinsicMetadataByIdRow) self._extid = Table(ExtIDRow) - self._stat_counters = defaultdict(int) - - def increment_counter(self, object_type: str, nb: int): - self._stat_counters[object_type] += nb - - def stat_counters(self) -> Iterable[ObjectCountRow]: - for (object_type, count) in self._stat_counters.items(): - yield ObjectCountRow(partition_key=0, object_type=object_type, count=count) ########################## # 'content' table @@ -186,7 +177,6 @@ def _content_add_finalize(self, content: ContentRow) -> None: self._contents.insert(content) - self.increment_counter("content", 1) def content_add_prepare(self, content: ContentRow): finalizer = functools.partial(self._content_add_finalize, content) @@ -248,7 +238,6 @@ def _skipped_content_add_finalize(self, content: SkippedContentRow) -> None: self._skipped_contents.insert(content) - self.increment_counter("skipped_content", 1) def skipped_content_add_prepare(self, content: SkippedContentRow): finalizer = functools.partial(self._skipped_content_add_finalize, content) @@ -293,7 +282,6 @@ def directory_add_one(self, directory: DirectoryRow) -> None: self._directories.insert(directory) - self.increment_counter("directory", 1) def directory_get_random(self) -> Optional[DirectoryRow]: return self._directories.get_random() @@ -334,7 +322,6 @@ def revision_add_one(self, revision: RevisionRow) -> None: self._revisions.insert(revision) - self.increment_counter("revision", 1) def revision_get_ids(self, revision_ids) -> Iterable[int]: for id_ in revision_ids: @@ -374,7 +361,6 @@ def release_add_one(self, release: ReleaseRow) -> None: self._releases.insert(release) - self.increment_counter("release", 1) def release_get(self, release_ids: List[str]) -> Iterable[ReleaseRow]: for id_ in release_ids: @@ -398,7 +384,6 @@ def snapshot_add_one(self, snapshot: SnapshotRow) -> None: self._snapshots.insert(snapshot) - self.increment_counter("snapshot", 1) def snapshot_get_random(self) -> Optional[SnapshotRow]: return self._snapshots.get_random() @@ -452,7 +437,6 @@ def origin_add_one(self, origin: OriginRow) -> None: self._origins.insert(origin) - self.increment_counter("origin", 1) def origin_get_by_sha1(self, sha1: bytes) -> Iterable[OriginRow]: return self._origins.get_from_partition_key((sha1,)) @@ -513,7 +497,6 @@ def origin_visit_add_one(self, visit: OriginVisitRow) -> None: self._origin_visits.insert(visit) - self.increment_counter("origin_visit", 1) def origin_visit_get_one( self, origin_url: str, visit_id: int @@ -558,7 +541,6 @@ def origin_visit_status_add_one(self, visit_update: OriginVisitStatusRow) -> None: self._origin_visit_statuses.insert(visit_update) - self.increment_counter("origin_visit_status", 1) def origin_visit_status_get_latest( self, origin: str, visit: int, @@ -588,7 +570,6 @@ def metadata_authority_add(self, authority: MetadataAuthorityRow): self._metadata_authorities.insert(authority) - self.increment_counter("metadata_authority", 1) def metadata_authority_get(self, type, url) -> Optional[MetadataAuthorityRow]: return self._metadata_authorities.get_from_primary_key((url, type)) @@ -599,7 +580,6 @@ def metadata_fetcher_add(self, fetcher: MetadataFetcherRow): self._metadata_fetchers.insert(fetcher) - self.increment_counter("metadata_fetcher", 1) def metadata_fetcher_get(self, name, version) -> Optional[MetadataAuthorityRow]: return self._metadata_fetchers.get_from_primary_key((name, version)) @@ -629,7 +609,6 @@ def raw_extrinsic_metadata_add(self, raw_extrinsic_metadata): self._raw_extrinsic_metadata.insert(raw_extrinsic_metadata) - self.increment_counter("raw_extrinsic_metadata", 1) def raw_extrinsic_metadata_get_after_date( self, @@ -682,7 +661,6 @@ ######################### def _extid_add_finalize(self, extid: ExtIDRow) -> None: self._extid.insert(extid) - self.increment_counter("extid", 1) def extid_add_prepare(self, extid: ExtIDRow): finalizer = functools.partial(self._extid_add_finalize, extid) diff --git a/swh/storage/tests/storage_tests.py b/swh/storage/tests/storage_tests.py --- a/swh/storage/tests/storage_tests.py +++ b/swh/storage/tests/storage_tests.py @@ -38,6 +38,7 @@ TargetType, ) from swh.storage import get_storage +from swh.storage.cassandra.storage import CassandraStorage from swh.storage.common import origin_url_to_sha1 as sha1 from swh.storage.exc import HashCollision, StorageArgumentException from swh.storage.interface import ListOrder, PagedResult, StorageInterface @@ -187,8 +188,9 @@ assert obj.ctime <= insertion_end_time assert obj == expected_cont - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["content"] == 1 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["content"] == 1 def test_content_add_from_lazy_content(self, swh_storage, sample_data): cont = sample_data.content @@ -221,8 +223,9 @@ assert obj.ctime <= insertion_end_time assert attr.evolve(obj, ctime=None).to_dict() == expected_cont.to_dict() - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["content"] == 1 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["content"] == 1 def test_content_get_data_missing(self, swh_storage, sample_data): cont, cont2 = sample_data.contents[:2] @@ -705,8 +708,9 @@ after_missing = list(swh_storage.directory_missing([directory.id])) assert after_missing == [] - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["directory"] == 1 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["directory"] == 1 def test_directory_add_twice(self, swh_storage, sample_data): directory = sample_data.directories[1] @@ -975,8 +979,9 @@ actual_result = swh_storage.revision_add([revision]) assert actual_result == {"revision:add": 0} - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["revision"] == 1 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["revision"] == 1 def test_revision_add_twice(self, swh_storage, sample_data): revision, revision2 = sample_data.revisions[:2] @@ -1376,8 +1381,9 @@ actual_result = swh_storage.release_add([release, release2]) assert actual_result == {"release:add": 0} - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["release"] == 2 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["release"] == 2 def test_release_add_no_author_date(self, swh_storage, sample_data): full_release = sample_data.release @@ -1482,8 +1488,9 @@ [("origin", origin) for origin in origins] ) - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["origin"] == len(origins) + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["origin"] == len(origins) def test_origin_add_twice(self, swh_storage, sample_data): origin, origin2 = sample_data.origins[:2] @@ -1921,11 +1928,11 @@ ] ) - swh_storage.refresh_stat_counters() - - stats = swh_storage.stat_counters() - assert stats["origin"] == len(origins) - assert stats["origin_visit"] == len(origins) * len(visits) + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + stats = swh_storage.stat_counters() + assert stats["origin"] == len(origins) + assert stats["origin_visit"] == len(origins) * len(visits) random_ovs = swh_storage.origin_visit_status_get_random(visit_type) assert random_ovs @@ -3122,8 +3129,9 @@ "next_branch": None, } - swh_storage.refresh_stat_counters() - assert swh_storage.stat_counters()["snapshot"] == 2 + if not isinstance(swh_storage, CassandraStorage): + swh_storage.refresh_stat_counters() + assert swh_storage.stat_counters()["snapshot"] == 2 def test_snapshot_add_many_incremental(self, swh_storage, sample_data): snapshot, _, complete_snapshot = sample_data.snapshots[:3] @@ -3623,6 +3631,8 @@ assert list(missing_snapshots) == [missing_snapshot.id] def test_stat_counters(self, swh_storage, sample_data): + if isinstance(swh_storage, CassandraStorage): + pytest.skip("Cassandra backend does not support stat counters") origin = sample_data.origin snapshot = sample_data.snapshot revision = sample_data.revision