diff --git a/swh/storage/cassandra/converters.py b/swh/storage/cassandra/converters.py --- a/swh/storage/cassandra/converters.py +++ b/swh/storage/cassandra/converters.py @@ -8,12 +8,15 @@ import attr +from typing import Dict + from swh.model.model import ( RevisionType, ObjectType, Revision, Release, ) - +from swh.model.hashutil import DEFAULT_ALGORITHMS from ..converters import git_headers_to_db, db_to_git_headers +from .common import Row def revision_to_db(revision: Revision) -> Revision: @@ -61,3 +64,13 @@ target_type=ObjectType(release.target_type), ) return release + + +def row_to_content_hashes(row: Row) -> Dict[str, bytes]: + """Convert cassandra row to a content hashes + + """ + hashes = {} + for algo in DEFAULT_ALGORITHMS: + hashes[algo] = getattr(row, algo) + return hashes diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -26,6 +26,7 @@ from .common import TOKEN_BEGIN, TOKEN_END from .converters import ( revision_to_db, revision_from_db, release_to_db, release_from_db, + row_to_content_hashes, ) from .cql import CqlRunner from .schema import HASH_ALGORITHMS @@ -93,7 +94,11 @@ algo, content.get_hash(algo)) if len(pks) > 1: # There are more than the one we just inserted. - raise HashCollision(algo, content.get_hash(algo), pks) + colliding_content_hashes = [ + row_to_content_hashes(pk) for pk in pks + ] + raise HashCollision( + algo, content.get_hash(algo), colliding_content_hashes) summary = { 'content:add': content_add, diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -92,7 +92,9 @@ hash_ = content.get_hash(algorithm) if hash_ in self._content_indexes[algorithm]\ and (algorithm not in {'blake2s256', 'sha256'}): - raise HashCollision(algorithm, hash_, key) + colliding_content_hashes = [content.hashes()] + raise HashCollision( + algorithm, hash_, colliding_content_hashes) for algorithm in DEFAULT_ALGORITHMS: hash_ = content.get_hash(algorithm) self._content_indexes[algorithm][hash_].add(key) diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -33,7 +33,9 @@ from .exc import StorageArgumentException, StorageDBError from .algos import diff from .metrics import timed, send_metric, process_metrics -from .utils import get_partition_bounds_bytes +from .utils import ( + get_partition_bounds_bytes, extract_collision_hash +) from .writer import JournalWriter @@ -158,14 +160,27 @@ except psycopg2.IntegrityError as e: if e.diag.sqlstate == '23505' and \ e.diag.table_name == 'content': - constraint_to_hash_name = { - 'content_pkey': 'sha1', - 'content_sha1_git_idx': 'sha1_git', - 'content_sha256_idx': 'sha256', + message_detail = e.diag.message_detail + if message_detail: + hash_name, hash_id = extract_collision_hash(message_detail) + collision_contents_hashes = [ + c.hashes() for c in content + if c.get_hash(hash_name) == hash_id + ] + else: + constraint_to_hash_name = { + 'content_pkey': 'sha1', + 'content_sha1_git_idx': 'sha1_git', + 'content_sha256_idx': 'sha256', } - colliding_hash_name = constraint_to_hash_name \ - .get(e.diag.constraint_name) - raise HashCollision(colliding_hash_name) from None + hash_name = constraint_to_hash_name \ + .get(e.diag.constraint_name) + hash_id = None + collision_contents_hashes = None + + raise HashCollision( + hash_name, hash_id, collision_contents_hashes + ) from None else: raise diff --git a/swh/storage/tests/test_utils.py b/swh/storage/tests/test_utils.py new file mode 100644 --- /dev/null +++ b/swh/storage/tests/test_utils.py @@ -0,0 +1,35 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from swh.model.hashutil import hash_to_bytes +from swh.storage.utils import extract_collision_hash + + +def test_extract_collision_hash(): + for msg, expected_result in [ + ( + 'Key (sha1)=(\\x34973274ccef6ab4dfaaf86599792fa9c3fe4689) ...', + ('sha1', hash_to_bytes( + '34973274ccef6ab4dfaaf86599792fa9c3fe4689')), + ), + ( + 'Key (sha1_git)=(\\x34973274ccef6ab4dfaaf86599792fa9c3fe4699) already exists', # noqa + ('sha1_git', hash_to_bytes( + '34973274ccef6ab4dfaaf86599792fa9c3fe4699')), + ), + ( + 'Key (sha256)=(\\x673650f936cb3b0a2f93ce09d81be10748b1b203c19e8176b4eefc1964a0cf3a) ...', # noqa + ('sha256', hash_to_bytes( + '673650f936cb3b0a2f93ce09d81be10748b1b203c19e8176b4eefc1964a0cf3a')) # noqa + ), + ( + 'Key (blake2s)=(\\xd5fe1939576527e42cfd76a9455a2432fe7f56669564577dd93c4280e76d661d) ...', # noqa + ('blake2s', hash_to_bytes( + 'd5fe1939576527e42cfd76a9455a2432fe7f56669564577dd93c4280e76d661d')) # noqa + ), + ]: + assert extract_collision_hash(msg) == expected_result + + assert extract_collision_hash('Nothing matching') is None diff --git a/swh/storage/utils.py b/swh/storage/utils.py --- a/swh/storage/utils.py +++ b/swh/storage/utils.py @@ -1,10 +1,14 @@ -# Copyright (C) 2019 The Software Heritage developers +# Copyright (C) 2019-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import re + from typing import Optional, Tuple +from swh.model.hashutil import hash_to_bytes + def _is_power_of_two(n: int) -> bool: return n > 0 and n & (n-1) == 0 @@ -40,3 +44,25 @@ end = None if i == n-1 \ else (partition_size*(i+1)).to_bytes(nb_bytes, 'big') return (start, end) + + +def extract_collision_hash(error_message: str) -> Optional[Tuple[str, bytes]]: + """Utilities to extract the hash information from a hash collision error. + + Hash collision error message are of the form: + 'Key ()=([^)]+)\)=\(\\x(?P[a-f0-9]+)\) \w*' + result = re.match(pattern, error_message) + if result: + hash_type = result.group('type') + hash_id = result.group('id') + return hash_type, hash_to_bytes(hash_id) + return None