Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/buffer.py
Show All 15 Lines | |||||
LObjectType = Literal[ | LObjectType = Literal[ | ||||
"content", | "content", | ||||
"skipped_content", | "skipped_content", | ||||
"directory", | "directory", | ||||
"revision", | "revision", | ||||
"release", | "release", | ||||
"snapshot", | "snapshot", | ||||
"extid", | "extid", | ||||
"origin", | |||||
"origin_visit_status", | |||||
"raw_extrinsic_metadata", | |||||
"metadata_fetcher", | |||||
"metadata_authority", | |||||
] | ] | ||||
OBJECT_TYPES: Tuple[LObjectType, ...] = ( | OBJECT_TYPES: Tuple[LObjectType, ...] = ( | ||||
"content", | "content", | ||||
"skipped_content", | "skipped_content", | ||||
"directory", | "directory", | ||||
"revision", | "revision", | ||||
"release", | "release", | ||||
"snapshot", | "snapshot", | ||||
"extid", | "extid", | ||||
"origin", | |||||
"origin_visit_status", | |||||
"raw_extrinsic_metadata", | |||||
"metadata_fetcher", | |||||
"metadata_authority", | |||||
) | ) | ||||
DEFAULT_BUFFER_THRESHOLDS: Dict[str, int] = { | DEFAULT_BUFFER_THRESHOLDS: Dict[str, int] = { | ||||
"content": 10000, | "content": 10000, | ||||
"content_bytes": 100 * 1024 * 1024, | "content_bytes": 100 * 1024 * 1024, | ||||
"skipped_content": 10000, | "skipped_content": 10000, | ||||
"directory": 25000, | "directory": 25000, | ||||
"revision": 100000, | "revision": 100000, | ||||
"release": 100000, | "release": 100000, | ||||
"snapshot": 25000, | "snapshot": 25000, | ||||
"extid": 10000, | "extid": 10000, | ||||
"origin": 100000, | |||||
"origin_visit_status": 10000, | |||||
vlorentz: hmm, they can be pretty big. You may want to divide this by 10 | |||||
Done Inline ActionsI wasn't sure, so ok douardda: I wasn't sure, so ok | |||||
"raw_extrinsic_metadata": 10000, | |||||
"metadata_fetcher": 100000, | |||||
"metadata_authority": 100000, | |||||
} | } | ||||
class BufferingProxyStorage: | class BufferingProxyStorage: | ||||
"""Storage implementation in charge of accumulating objects prior to | """Storage implementation in charge of accumulating objects prior to | ||||
discussing with the "main" storage. | discussing with the "main" storage. | ||||
Deduplicates values based on a tuple of keys depending on the object type. | Deduplicates values based on a tuple of keys depending on the object type. | ||||
▲ Show 20 Lines • Show All 132 Lines • Show Last 20 Lines |
hmm, they can be pretty big. You may want to divide this by 10