Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/buffer.py
Show First 20 Lines • Show All 57 Lines • ▼ Show 20 Lines | def __getattr__(self, key): | ||||
self.object_add, object_type=object_type | self.object_add, object_type=object_type | ||||
) | ) | ||||
return getattr(self.storage, key) | return getattr(self.storage, key) | ||||
def content_add(self, content: Iterable[Dict]) -> Dict: | def content_add(self, content: Iterable[Dict]) -> Dict: | ||||
"""Enqueue contents to write to the storage. | """Enqueue contents to write to the storage. | ||||
Following policies apply: | Following policies apply: | ||||
- First, check if the queue's threshold is hit. If it is flush content | |||||
to the storage. | |||||
- If not, check if the total size of enqueued contents's threshold is | - First, check if the queue's threshold is hit. | ||||
hit. If it is flush content to the storage. | If it is flush content to the storage. | ||||
- If not, check if the total size of enqueued contents's | |||||
threshold is hit. If it is flush content to the storage. | |||||
""" | """ | ||||
s = self.object_add(content, object_type='content') | s = self.object_add(content, object_type='content') | ||||
if not s: | if not s: | ||||
q = self._objects['content'] | q = self._objects['content'] | ||||
total_size = sum(c['length'] for c in q) | total_size = sum(c['length'] for c in q) | ||||
if total_size >= self.min_batch_size['content_bytes']: | if total_size >= self.min_batch_size['content_bytes']: | ||||
return self.flush(['content']) | return self.flush(['content']) | ||||
Show All 30 Lines |