Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/buffer.py
Show First 20 Lines • Show All 94 Lines • ▼ Show 20 Lines | class BufferingProxyStorage: | ||||
def skipped_content_add(self, content: Iterable[Content]) -> Dict: | def skipped_content_add(self, content: Iterable[Content]) -> Dict: | ||||
return self.object_add( | return self.object_add( | ||||
content, | content, | ||||
object_type="skipped_content", | object_type="skipped_content", | ||||
keys=["sha1", "sha1_git", "sha256", "blake2s256"], | keys=["sha1", "sha1_git", "sha256", "blake2s256"], | ||||
) | ) | ||||
def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict: | def flush(self, object_types: Optional[Iterable[str]] = None) -> Dict: | ||||
vlorentz: `summary: Dict[str, int]` | |||||
Done Inline Actionsmmm, right! I missed the comment was wrong ;) ardumont: mmm, right! I missed the comment was wrong ;) | |||||
summary: Dict[str, int] = self.storage.flush(object_types) | |||||
if object_types is None: | if object_types is None: | ||||
object_types = self.object_types | object_types = self.object_types | ||||
summary = {} # type: Dict[str, Dict] | |||||
for object_type in object_types: | for object_type in object_types: | ||||
buffer_ = self._objects[object_type] | buffer_ = self._objects[object_type] | ||||
batches = grouper(buffer_.values(), n=self.min_batch_size[object_type]) | batches = grouper(buffer_.values(), n=self.min_batch_size[object_type]) | ||||
for batch in batches: | for batch in batches: | ||||
add_fn = getattr(self.storage, "%s_add" % object_type) | add_fn = getattr(self.storage, "%s_add" % object_type) | ||||
s = add_fn(batch) | s = add_fn(batch) | ||||
summary = {k: v + summary.get(k, 0) for k, v in s.items()} | summary = {k: v + summary.get(k, 0) for k, v in s.items()} | ||||
buffer_.clear() | buffer_.clear() | ||||
Show All 38 Lines |
summary: Dict[str, int]