Page MenuHomeSoftware Heritage
Files F7123977

D6150.id22254.diff
No OneTemporary
Actions

D6150.id22254.diff
View Options

diff --git a/swh/storage/cassandra/cql.py b/swh/storage/cassandra/cql.py
--- a/swh/storage/cassandra/cql.py
+++ b/swh/storage/cassandra/cql.py
@@ -62,7 +62,6 @@
ExtIDRow,
MetadataAuthorityRow,
MetadataFetcherRow,
- ObjectCountRow,
OriginRow,
OriginVisitRow,
OriginVisitStatusRow,
@@ -302,24 +301,13 @@
) -> ResultSet:
return execute_concurrent_with_args(self._session, statement, args_list)
- @_prepared_statement(
- "UPDATE object_count SET count = count + ? "
- "WHERE partition_key = 0 AND object_type = ?"
- )
- def _increment_counter(
- self, object_type: str, nb: int, *, statement: PreparedStatement
- ) -> None:
- self._execute_with_retries(statement, [nb, object_type])
-
def _add_one(self, statement, obj: BaseRow) -> None:
- self._increment_counter(obj.TABLE, 1)
self._execute_with_retries(statement, dataclasses.astuple(obj))
def _add_many(self, statement, objs: Sequence[BaseRow]) -> None:
tables = {obj.TABLE for obj in objs}
assert len(tables) == 1, f"Cannot insert to multiple tables: {tables}"
(table,) = tables
- self._increment_counter(table, len(objs))
self._execute_many_with_retries(statement, list(map(dataclasses.astuple, objs)))
_T = TypeVar("_T", bound=BaseRow)
@@ -354,7 +342,6 @@
"""Returned currified by content_add_prepare, to be called when the
content row should be added to the primary table."""
self._execute_with_retries(statement, None)
- self._increment_counter("content", 1)
@_prepared_insert_statement(ContentRow)
def content_add_prepare(
@@ -508,7 +495,6 @@
"""Returned currified by skipped_content_add_prepare, to be called
when the content row should be added to the primary table."""
self._execute_with_retries(statement, None)
- self._increment_counter("skipped_content", 1)
@_prepared_insert_statement(SkippedContentRow)
def skipped_content_add_prepare(
@@ -1295,7 +1281,6 @@
"""Returned currified by extid_add_prepare, to be called when the
extid row should be added to the primary table."""
self._execute_with_retries(statement, None)
- self._increment_counter("extid", 1)
@_prepared_insert_statement(ExtIDRow)
def extid_add_prepare(
@@ -1407,7 +1392,3 @@
@_prepared_statement("SELECT uuid() FROM revision LIMIT 1;")
def check_read(self, *, statement):
self._execute_with_retries(statement, [])
-
- @_prepared_select_statement(ObjectCountRow, "WHERE partition_key=0")
- def stat_counters(self, *, statement) -> Iterable[ObjectCountRow]:
- return map(ObjectCountRow.from_dict, self._execute_with_retries(statement, []))
diff --git a/swh/storage/cassandra/model.py b/swh/storage/cassandra/model.py
--- a/swh/storage/cassandra/model.py
+++ b/swh/storage/cassandra/model.py
@@ -300,17 +300,6 @@
authority_url: str
-@dataclasses.dataclass
-class ObjectCountRow(BaseRow):
- TABLE = "object_count"
- PARTITION_KEY = ("partition_key",)
- CLUSTERING_KEY = ("object_type",)
-
- partition_key: int
- object_type: str
- count: int
-
-
@dataclasses.dataclass
class ExtIDRow(BaseRow):
TABLE = "extid"
diff --git a/swh/storage/cassandra/schema.py b/swh/storage/cassandra/schema.py
--- a/swh/storage/cassandra/schema.py
+++ b/swh/storage/cassandra/schema.py
@@ -267,13 +267,6 @@
PRIMARY KEY ((id))
);""",
"""
-CREATE TABLE IF NOT EXISTS object_count (
- partition_key smallint, -- Constant, must always be 0
- object_type ascii,
- count counter,
- PRIMARY KEY ((partition_key), object_type)
-);""",
- """
CREATE TABLE IF NOT EXISTS extid (
extid_type ascii,
extid blob,
@@ -319,7 +312,6 @@
"origin_visit",
"origin",
"raw_extrinsic_metadata",
- "object_count",
"origin_visit_status",
"metadata_authority",
"metadata_fetcher",
diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py
--- a/swh/storage/cassandra/storage.py
+++ b/swh/storage/cassandra/storage.py
@@ -1290,20 +1290,7 @@
return None
def stat_counters(self):
- rows = self._cql_runner.stat_counters()
- keys = (
- "content",
- "directory",
- "origin",
- "origin_visit",
- "release",
- "revision",
- "skipped_content",
- "snapshot",
- )
- stats = {key: 0 for key in keys}
- stats.update({row.object_type: row.count for row in rows})
- return stats
+ raise NotImplementedError()
def refresh_stat_counters(self):
pass
diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py
--- a/swh/storage/in_memory.py
+++ b/swh/storage/in_memory.py
@@ -34,7 +34,6 @@
ExtIDRow,
MetadataAuthorityRow,
MetadataFetcherRow,
- ObjectCountRow,
OriginRow,
OriginVisitRow,
OriginVisitStatusRow,
@@ -171,14 +170,6 @@
self._raw_extrinsic_metadata = Table(RawExtrinsicMetadataRow)
self._raw_extrinsic_metadata_by_id = Table(RawExtrinsicMetadataByIdRow)
self._extid = Table(ExtIDRow)
- self._stat_counters = defaultdict(int)
-
- def increment_counter(self, object_type: str, nb: int):
- self._stat_counters[object_type] += nb
-
- def stat_counters(self) -> Iterable[ObjectCountRow]:
- for (object_type, count) in self._stat_counters.items():
- yield ObjectCountRow(partition_key=0, object_type=object_type, count=count)
##########################
# 'content' table
@@ -186,7 +177,6 @@
def _content_add_finalize(self, content: ContentRow) -> None:
self._contents.insert(content)
- self.increment_counter("content", 1)
def content_add_prepare(self, content: ContentRow):
finalizer = functools.partial(self._content_add_finalize, content)
@@ -248,7 +238,6 @@
def _skipped_content_add_finalize(self, content: SkippedContentRow) -> None:
self._skipped_contents.insert(content)
- self.increment_counter("skipped_content", 1)
def skipped_content_add_prepare(self, content: SkippedContentRow):
finalizer = functools.partial(self._skipped_content_add_finalize, content)
@@ -293,7 +282,6 @@
def directory_add_one(self, directory: DirectoryRow) -> None:
self._directories.insert(directory)
- self.increment_counter("directory", 1)
def directory_get_random(self) -> Optional[DirectoryRow]:
return self._directories.get_random()
@@ -334,7 +322,6 @@
def revision_add_one(self, revision: RevisionRow) -> None:
self._revisions.insert(revision)
- self.increment_counter("revision", 1)
def revision_get_ids(self, revision_ids) -> Iterable[int]:
for id_ in revision_ids:
@@ -374,7 +361,6 @@
def release_add_one(self, release: ReleaseRow) -> None:
self._releases.insert(release)
- self.increment_counter("release", 1)
def release_get(self, release_ids: List[str]) -> Iterable[ReleaseRow]:
for id_ in release_ids:
@@ -398,7 +384,6 @@
def snapshot_add_one(self, snapshot: SnapshotRow) -> None:
self._snapshots.insert(snapshot)
- self.increment_counter("snapshot", 1)
def snapshot_get_random(self) -> Optional[SnapshotRow]:
return self._snapshots.get_random()
@@ -452,7 +437,6 @@
def origin_add_one(self, origin: OriginRow) -> None:
self._origins.insert(origin)
- self.increment_counter("origin", 1)
def origin_get_by_sha1(self, sha1: bytes) -> Iterable[OriginRow]:
return self._origins.get_from_partition_key((sha1,))
@@ -513,7 +497,6 @@
def origin_visit_add_one(self, visit: OriginVisitRow) -> None:
self._origin_visits.insert(visit)
- self.increment_counter("origin_visit", 1)
def origin_visit_get_one(
self, origin_url: str, visit_id: int
@@ -558,7 +541,6 @@
def origin_visit_status_add_one(self, visit_update: OriginVisitStatusRow) -> None:
self._origin_visit_statuses.insert(visit_update)
- self.increment_counter("origin_visit_status", 1)
def origin_visit_status_get_latest(
self, origin: str, visit: int,
@@ -588,7 +570,6 @@
def metadata_authority_add(self, authority: MetadataAuthorityRow):
self._metadata_authorities.insert(authority)
- self.increment_counter("metadata_authority", 1)
def metadata_authority_get(self, type, url) -> Optional[MetadataAuthorityRow]:
return self._metadata_authorities.get_from_primary_key((url, type))
@@ -599,7 +580,6 @@
def metadata_fetcher_add(self, fetcher: MetadataFetcherRow):
self._metadata_fetchers.insert(fetcher)
- self.increment_counter("metadata_fetcher", 1)
def metadata_fetcher_get(self, name, version) -> Optional[MetadataAuthorityRow]:
return self._metadata_fetchers.get_from_primary_key((name, version))
@@ -629,7 +609,6 @@
def raw_extrinsic_metadata_add(self, raw_extrinsic_metadata):
self._raw_extrinsic_metadata.insert(raw_extrinsic_metadata)
- self.increment_counter("raw_extrinsic_metadata", 1)
def raw_extrinsic_metadata_get_after_date(
self,
@@ -682,7 +661,6 @@
#########################
def _extid_add_finalize(self, extid: ExtIDRow) -> None:
self._extid.insert(extid)
- self.increment_counter("extid", 1)
def extid_add_prepare(self, extid: ExtIDRow):
finalizer = functools.partial(self._extid_add_finalize, extid)
diff --git a/swh/storage/tests/storage_tests.py b/swh/storage/tests/storage_tests.py
--- a/swh/storage/tests/storage_tests.py
+++ b/swh/storage/tests/storage_tests.py
@@ -38,6 +38,7 @@
TargetType,
)
from swh.storage import get_storage
+from swh.storage.cassandra.storage import CassandraStorage
from swh.storage.common import origin_url_to_sha1 as sha1
from swh.storage.exc import HashCollision, StorageArgumentException
from swh.storage.interface import ListOrder, PagedResult, StorageInterface
@@ -187,8 +188,9 @@
assert obj.ctime <= insertion_end_time
assert obj == expected_cont
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["content"] == 1
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["content"] == 1
def test_content_add_from_lazy_content(self, swh_storage, sample_data):
cont = sample_data.content
@@ -221,8 +223,9 @@
assert obj.ctime <= insertion_end_time
assert attr.evolve(obj, ctime=None).to_dict() == expected_cont.to_dict()
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["content"] == 1
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["content"] == 1
def test_content_get_data_missing(self, swh_storage, sample_data):
cont, cont2 = sample_data.contents[:2]
@@ -705,8 +708,9 @@
after_missing = list(swh_storage.directory_missing([directory.id]))
assert after_missing == []
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["directory"] == 1
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["directory"] == 1
def test_directory_add_twice(self, swh_storage, sample_data):
directory = sample_data.directories[1]
@@ -975,8 +979,9 @@
actual_result = swh_storage.revision_add([revision])
assert actual_result == {"revision:add": 0}
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["revision"] == 1
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["revision"] == 1
def test_revision_add_twice(self, swh_storage, sample_data):
revision, revision2 = sample_data.revisions[:2]
@@ -1376,8 +1381,9 @@
actual_result = swh_storage.release_add([release, release2])
assert actual_result == {"release:add": 0}
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["release"] == 2
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["release"] == 2
def test_release_add_no_author_date(self, swh_storage, sample_data):
full_release = sample_data.release
@@ -1482,8 +1488,9 @@
[("origin", origin) for origin in origins]
)
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["origin"] == len(origins)
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["origin"] == len(origins)
def test_origin_add_twice(self, swh_storage, sample_data):
origin, origin2 = sample_data.origins[:2]
@@ -1921,11 +1928,11 @@
]
)
- swh_storage.refresh_stat_counters()
-
- stats = swh_storage.stat_counters()
- assert stats["origin"] == len(origins)
- assert stats["origin_visit"] == len(origins) * len(visits)
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ stats = swh_storage.stat_counters()
+ assert stats["origin"] == len(origins)
+ assert stats["origin_visit"] == len(origins) * len(visits)
random_ovs = swh_storage.origin_visit_status_get_random(visit_type)
assert random_ovs
@@ -3122,8 +3129,9 @@
"next_branch": None,
}
- swh_storage.refresh_stat_counters()
- assert swh_storage.stat_counters()["snapshot"] == 2
+ if not isinstance(swh_storage, CassandraStorage):
+ swh_storage.refresh_stat_counters()
+ assert swh_storage.stat_counters()["snapshot"] == 2
def test_snapshot_add_many_incremental(self, swh_storage, sample_data):
snapshot, _, complete_snapshot = sample_data.snapshots[:3]
@@ -3623,6 +3631,8 @@
assert list(missing_snapshots) == [missing_snapshot.id]
def test_stat_counters(self, swh_storage, sample_data):
+ if isinstance(swh_storage, CassandraStorage):
+ pytest.skip("Cassandra backend does not support stat counters")
origin = sample_data.origin
snapshot = sample_data.snapshot
revision = sample_data.revision

File Metadata

Mime Type
text/plain
Expires
Dec 20 2024, 7:40 AM (11 w, 3 d ago)
Storage Engine
blob
Storage Format
Raw Data
Storage Handle
3227487

Event Timeline

About · Developer information · Legal