Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/cql.py
| Show First 20 Lines • Show All 433 Lines • ▼ Show 20 Lines | ) -> Optional[ContentRow]: | ||||
| ) | ) | ||||
| ) | ) | ||||
| assert len(rows) <= 1 | assert len(rows) <= 1 | ||||
| if rows: | if rows: | ||||
| return ContentRow(**rows[0]) | return ContentRow(**rows[0]) | ||||
| else: | else: | ||||
| return None | return None | ||||
| def content_missing_from_hashes( | def content_missing_from_all_hashes( | ||||
| self, contents_hashes: List[Dict[str, bytes]] | self, contents_hashes: List[Dict[str, bytes]] | ||||
| ) -> Iterator[Dict[str, bytes]]: | ) -> Iterator[Dict[str, bytes]]: | ||||
| for group in grouper(contents_hashes, PARTITION_KEY_RESTRICTION_MAX_SIZE): | for group in grouper(contents_hashes, PARTITION_KEY_RESTRICTION_MAX_SIZE): | ||||
| group = list(group) | group = list(group) | ||||
| # Get all contents that share a sha256 with one of the contents in the group | # Get all contents that share a sha256 with one of the contents in the group | ||||
| present = set( | present = set( | ||||
| self._content_get_hashes_from_sha256( | self._content_get_hashes_from_sha256( | ||||
| [content["sha256"] for content in group] | [content["sha256"] for content in group] | ||||
| ) | ) | ||||
| ) | ) | ||||
| for content in group: | for content in group: | ||||
| for algo in HASH_ALGORITHMS: | for algo in HASH_ALGORITHMS: | ||||
| assert content.get(algo) is not None, ( | assert content.get(algo) is not None, ( | ||||
| "content_missing_from_hashes must not be called with " | "content_missing_from_all_hashes must not be called with " | ||||
| "partial hashes." | "partial hashes." | ||||
| ) | ) | ||||
| if tuple(content[algo] for algo in HASH_ALGORITHMS) not in present: | if tuple(content[algo] for algo in HASH_ALGORITHMS) not in present: | ||||
| yield content | yield content | ||||
| @_prepared_statement( | @_prepared_statement( | ||||
| f"SELECT {', '.join(HASH_ALGORITHMS)} FROM content WHERE sha256 IN ?" | f"SELECT {', '.join(HASH_ALGORITHMS)} FROM content WHERE sha256 IN ?" | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 1,022 Lines • Show Last 20 Lines | |||||