Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/buffer.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from functools import partial | from functools import partial | ||||
from typing import Dict, Iterable, Mapping, Sequence, Tuple | from typing import Dict, Iterable, Mapping, Sequence, Tuple | ||||
from typing_extensions import Literal | from typing_extensions import Literal | ||||
from swh.core.utils import grouper | from swh.core.utils import grouper | ||||
from swh.model.model import BaseModel, Content, SkippedContent | from swh.model.model import BaseModel, Content, SkippedContent | ||||
from swh.storage import get_storage | from swh.storage import get_storage | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
LObjectType = Literal[ | LObjectType = Literal[ | ||||
"content", "skipped_content", "directory", "revision", "release", "snapshot" | "content", | ||||
"skipped_content", | |||||
"directory", | |||||
"revision", | |||||
"release", | |||||
"snapshot", | |||||
"extid", | |||||
] | ] | ||||
OBJECT_TYPES: Tuple[LObjectType, ...] = ( | OBJECT_TYPES: Tuple[LObjectType, ...] = ( | ||||
"content", | "content", | ||||
"skipped_content", | "skipped_content", | ||||
"directory", | "directory", | ||||
"revision", | "revision", | ||||
"release", | "release", | ||||
"snapshot", | "snapshot", | ||||
"extid", | |||||
) | ) | ||||
DEFAULT_BUFFER_THRESHOLDS: Dict[str, int] = { | DEFAULT_BUFFER_THRESHOLDS: Dict[str, int] = { | ||||
"content": 10000, | "content": 10000, | ||||
"content_bytes": 100 * 1024 * 1024, | "content_bytes": 100 * 1024 * 1024, | ||||
"skipped_content": 10000, | "skipped_content": 10000, | ||||
"directory": 25000, | "directory": 25000, | ||||
"revision": 100000, | "revision": 100000, | ||||
"release": 100000, | "release": 100000, | ||||
"snapshot": 25000, | "snapshot": 25000, | ||||
"extid": 10000, | |||||
} | } | ||||
class BufferingProxyStorage: | class BufferingProxyStorage: | ||||
"""Storage implementation in charge of accumulating objects prior to | """Storage implementation in charge of accumulating objects prior to | ||||
discussing with the "main" storage. | discussing with the "main" storage. | ||||
Deduplicates values based on a tuple of keys depending on the object type. | Deduplicates values based on a tuple of keys depending on the object type. | ||||
▲ Show 20 Lines • Show All 131 Lines • Show Last 20 Lines |