diff --git a/swh/storage/cassandra/converters.py b/swh/storage/cassandra/converters.py --- a/swh/storage/cassandra/converters.py +++ b/swh/storage/cassandra/converters.py @@ -3,15 +3,14 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from copy import deepcopy import json +from typing import Any, Dict, List import attr -from copy import deepcopy -from typing import Dict - from swh.model.model import ( - RevisionType, ObjectType, Revision, Release, + RevisionType, ObjectType, Revision, Release, Sha1Git, ) from swh.model.hashutil import DEFAULT_ALGORITHMS @@ -19,16 +18,12 @@ from .common import Row -class CassObject(dict): - __getattr__ = dict.__getitem__ - - -def revision_to_db(revision: Revision) -> CassObject: +def revision_to_db(revision: Revision) -> Dict[str, Any]: # we use a deepcopy of the dict because we do not want to recurse the # Model->dict conversion (to keep Timestamp & al. entities), BUT we do not # want to modify original metadata (embedded in the Model entity), so we # non-recursively convert it as a dict but make a deep copy. - db_revision = CassObject(deepcopy(attr.asdict(revision, recurse=False))) + db_revision = deepcopy(attr.asdict(revision, recurse=False)) metadata = revision.metadata if metadata and 'extra_headers' in metadata: db_revision['metadata']['extra_headers'] = git_headers_to_db( @@ -38,25 +33,33 @@ return db_revision -def revision_from_db(**kwargs) -> Revision: - kwargs['metadata'] = metadata = json.loads(kwargs['metadata']) +def revision_from_db(db_revision: Row, parents: List[Sha1Git]) -> Revision: + revision = db_revision._asdict() # type: ignore + metadata = json.loads(revision.pop('metadata', None)) if metadata and 'extra_headers' in metadata: extra_headers = db_to_git_headers( metadata['extra_headers']) metadata['extra_headers'] = extra_headers - kwargs['type'] = RevisionType(kwargs['type']) - return Revision(**kwargs) + return Revision( + parents=parents, + type=RevisionType(revision.pop('type')), + metadata=metadata, + **revision, + ) -def release_to_db(release: Release) -> CassObject: - db_release = CassObject(attr.asdict(release, recurse=False)) - db_release['target_type'] = release.target_type.value +def release_to_db(release: Release) -> Dict[str, Any]: + db_release = attr.asdict(release, recurse=False) + db_release['target_type'] = db_release['target_type'].value return db_release -def release_from_db(**kwargs) -> Release: - kwargs['target_type'] = ObjectType(kwargs['target_type']) - return Release(**kwargs) +def release_from_db(db_release: Row) -> Release: + release = db_release._asdict() # type: ignore + return Release( + target_type=ObjectType(release.pop('target_type')), + **release, + ) def row_to_content_hashes(row: Row) -> Dict[str, bytes]: diff --git a/swh/storage/cassandra/cql.py b/swh/storage/cassandra/cql.py --- a/swh/storage/cassandra/cql.py +++ b/swh/storage/cassandra/cql.py @@ -364,7 +364,9 @@ @_prepared_insert_statement('revision', _revision_keys) def revision_add_one(self, revision: Dict[str, Any], *, statement) -> None: - self._add_one(statement, 'revision', revision, self._revision_keys) + self._execute_with_retries( + statement, [revision[key] for key in self._revision_keys]) + self._increment_counter('revision', 1) @_prepared_statement('SELECT id FROM revision WHERE id IN ?') def revision_get_ids(self, revision_ids, *, statement) -> ResultSet: @@ -413,7 +415,9 @@ @_prepared_insert_statement('release', _release_keys) def release_add_one(self, release: Dict[str, Any], *, statement) -> None: - self._add_one(statement, 'release', release, self._release_keys) + self._execute_with_retries( + statement, [release[key] for key in self._release_keys]) + self._increment_counter('release', 1) @_prepared_statement('SELECT * FROM release WHERE id in ?') def release_get(self, release_ids: List[str], *, statement) -> None: diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -443,7 +443,7 @@ # parent_rank is the clustering key, so results are already # sorted by rank. parents = [row.parent_id for row in parent_rows] - rev = revision_from_db(**row._asdict(), parents=parents) + rev = revision_from_db(row, parents=parents) revs[rev.id] = rev.to_dict() for rev_id in revisions: @@ -479,8 +479,7 @@ if short: yield (row.id, parents) else: - rev = revision_from_db( - **row._asdict(), parents=parents) + rev = revision_from_db(row, parents=parents) yield rev.to_dict() yield from self._get_parent_revs(parents, seen, limit, short) @@ -514,7 +513,7 @@ rows = self._cql_runner.release_get(releases) rels = {} for row in rows: - release = release_from_db(**row._asdict()) + release = release_from_db(row) rels[row.id] = release.to_dict() for rel_id in releases: