Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/cql.py
Show First 20 Lines • Show All 388 Lines • ▼ Show 20 Lines | ) -> Optional[ContentRow]: | ||||
) | ) | ||||
) | ) | ||||
assert len(rows) <= 1 | assert len(rows) <= 1 | ||||
if rows: | if rows: | ||||
return ContentRow(**rows[0]) | return ContentRow(**rows[0]) | ||||
else: | else: | ||||
return None | return None | ||||
def content_missing_from_hashes( | def content_missing_from_all_hashes( | ||||
self, contents_hashes: List[Dict[str, bytes]] | self, contents_hashes: List[Dict[str, bytes]] | ||||
) -> Iterator[Dict[str, bytes]]: | ) -> Iterator[Dict[str, bytes]]: | ||||
for group in grouper(contents_hashes, PARTITION_KEY_RESTRICTION_MAX_SIZE): | for group in grouper(contents_hashes, PARTITION_KEY_RESTRICTION_MAX_SIZE): | ||||
group = list(group) | group = list(group) | ||||
# Get all contents that share a sha256 with one of the contents in the group | # Get all contents that share a sha256 with one of the contents in the group | ||||
present = set( | present = set( | ||||
self._content_get_hashes_from_sha256( | self._content_get_hashes_from_sha256( | ||||
[content["sha256"] for content in group] | [content["sha256"] for content in group] | ||||
) | ) | ||||
) | ) | ||||
for content in group: | for content in group: | ||||
for algo in HASH_ALGORITHMS: | for algo in HASH_ALGORITHMS: | ||||
assert content.get(algo) is not None, ( | assert content.get(algo) is not None, ( | ||||
"content_missing_from_hashes must not be called with " | "content_missing_from_all_hashes must not be called with " | ||||
"partial hashes." | "partial hashes." | ||||
) | ) | ||||
if tuple(content[algo] for algo in HASH_ALGORITHMS) not in present: | if tuple(content[algo] for algo in HASH_ALGORITHMS) not in present: | ||||
yield content | yield content | ||||
@_prepared_statement( | @_prepared_statement( | ||||
f"SELECT {', '.join(HASH_ALGORITHMS)} FROM content WHERE sha256 IN ?" | f"SELECT {', '.join(HASH_ALGORITHMS)} FROM content WHERE sha256 IN ?" | ||||
) | ) | ||||
▲ Show 20 Lines • Show All 1,030 Lines • Show Last 20 Lines |