Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/buffer.py
Show First 20 Lines • Show All 84 Lines • ▼ Show 20 Lines | def __getattr__(self, key: str): | ||||
if key.endswith("_add"): | if key.endswith("_add"): | ||||
object_type = key.rsplit("_", 1)[0] | object_type = key.rsplit("_", 1)[0] | ||||
if object_type in OBJECT_TYPES: | if object_type in OBJECT_TYPES: | ||||
return partial(self.object_add, object_type=object_type, keys=["id"],) | return partial(self.object_add, object_type=object_type, keys=["id"],) | ||||
if key == "storage": | if key == "storage": | ||||
raise AttributeError(key) | raise AttributeError(key) | ||||
return getattr(self.storage, key) | return getattr(self.storage, key) | ||||
def content_add(self, contents: Sequence[Content]) -> Dict: | def content_add(self, contents: Sequence[Content]) -> Dict[str, int]: | ||||
"""Push contents to write to the storage in the buffer. | """Push contents to write to the storage in the buffer. | ||||
Following policies apply: | Following policies apply: | ||||
- if the buffer's threshold is hit, flush content to the storage. | - if the buffer's threshold is hit, flush content to the storage. | ||||
- otherwise, if the total size of buffered contents's threshold is hit, | - otherwise, if the total size of buffered contents's threshold is hit, | ||||
flush content to the storage. | flush content to the storage. | ||||
""" | """ | ||||
stats = self.object_add( | stats = self.object_add( | ||||
contents, | contents, | ||||
object_type="content", | object_type="content", | ||||
keys=["sha1", "sha1_git", "sha256", "blake2s256"], | keys=["sha1", "sha1_git", "sha256", "blake2s256"], | ||||
) | ) | ||||
if not stats: # We did not flush already | if not stats: # We did not flush already | ||||
self._contents_size += sum(c.length for c in contents) | self._contents_size += sum(c.length for c in contents) | ||||
if self._contents_size >= self._buffer_thresholds["content_bytes"]: | if self._contents_size >= self._buffer_thresholds["content_bytes"]: | ||||
return self.flush(["content"]) | return self.flush(["content"]) | ||||
return stats | return stats | ||||
def skipped_content_add(self, contents: Sequence[SkippedContent]) -> Dict: | def skipped_content_add(self, contents: Sequence[SkippedContent]) -> Dict[str, int]: | ||||
return self.object_add( | return self.object_add( | ||||
contents, | contents, | ||||
object_type="skipped_content", | object_type="skipped_content", | ||||
keys=["sha1", "sha1_git", "sha256", "blake2s256"], | keys=["sha1", "sha1_git", "sha256", "blake2s256"], | ||||
) | ) | ||||
def object_add( | def object_add( | ||||
self, | self, | ||||
▲ Show 20 Lines • Show All 60 Lines • Show Last 20 Lines |