diff --git a/swh/storage/cassandra/converters.py b/swh/storage/cassandra/converters.py --- a/swh/storage/cassandra/converters.py +++ b/swh/storage/cassandra/converters.py @@ -8,7 +8,7 @@ import attr from copy import deepcopy -from typing import Any, Dict, Tuple +from typing import cast, Any, Dict, Optional, Tuple from cassandra.cluster import ResultSet @@ -22,6 +22,7 @@ ) from swh.model.hashutil import DEFAULT_ALGORITHMS +from swh.storage.utils import map_optional from .common import Row @@ -35,7 +36,9 @@ extra_headers = revision.extra_headers if not extra_headers and metadata and "extra_headers" in metadata: extra_headers = db_revision["metadata"].pop("extra_headers") - db_revision["metadata"] = json.dumps(db_revision["metadata"]) + db_revision["metadata"] = json.dumps( + map_optional(dict, cast(Optional[Dict], db_revision["metadata"])) + ) db_revision["extra_headers"] = extra_headers db_revision["type"] = db_revision["type"].value return db_revision diff --git a/swh/storage/cassandra/cql.py b/swh/storage/cassandra/cql.py --- a/swh/storage/cassandra/cql.py +++ b/swh/storage/cassandra/cql.py @@ -44,6 +44,8 @@ Origin, ) +from swh.storage.utils import map_optional + from .common import Row, TOKEN_BEGIN, TOKEN_END, hash_url from .schema import CREATE_TABLES_QUERIES, HASH_ALGORITHMS from .. import extrinsic_metadata @@ -779,7 +781,7 @@ assert self._origin_visit_status_keys[-1] == "metadata" keys = self._origin_visit_status_keys - metadata = json.dumps(visit_update.metadata) + metadata = json.dumps(map_optional(dict, visit_update.metadata)) self._execute_with_retries( statement, [getattr(visit_update, key) for key in keys[:-1]] + [metadata] ) diff --git a/swh/storage/db.py b/swh/storage/db.py --- a/swh/storage/db.py +++ b/swh/storage/db.py @@ -9,10 +9,16 @@ from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from swh.core.db import BaseDb -from swh.core.db.db_utils import stored_procedure, jsonize +from swh.core.db.db_utils import stored_procedure, jsonize as _jsonize from swh.core.db.db_utils import execute_values_generator from swh.model.model import OriginVisit, OriginVisitStatus, SHA1_SIZE +from .utils import map_optional + + +def jsonize(d): + return _jsonize(map_optional(dict, d)) + class Db(BaseDb): """Proxy to the SWH DB, with wrappers around stored procedures diff --git a/swh/storage/utils.py b/swh/storage/utils.py --- a/swh/storage/utils.py +++ b/swh/storage/utils.py @@ -6,7 +6,7 @@ import re from datetime import datetime, timezone -from typing import Dict, Optional, Tuple +from typing import Callable, Dict, Optional, Tuple, TypeVar from swh.model.hashutil import hash_to_bytes, hash_to_hex, DEFAULT_ALGORITHMS @@ -15,6 +15,18 @@ return datetime.now(tz=timezone.utc) +T1 = TypeVar("T1") +T2 = TypeVar("T2") + + +def map_optional(f: Callable[[T1], T2], x: Optional[T1]) -> Optional[T2]: + """Equivalent to `None if x is None else f(x)`.""" + if x is None: + return None + else: + return f(x) + + def _is_power_of_two(n: int) -> bool: return n > 0 and n & (n - 1) == 0