diff --git a/swh/objstorage/objstorage_pathslicing.py b/swh/objstorage/objstorage_pathslicing.py --- a/swh/objstorage/objstorage_pathslicing.py +++ b/swh/objstorage/objstorage_pathslicing.py @@ -3,6 +3,7 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +import functools import os import gzip import tempfile @@ -226,6 +227,17 @@ return obj_id + def add_stream(self, content_iter, obj_id, check_presence=True): + if check_presence and obj_id in self: + return obj_id + + hex_obj_id = hashutil.hash_to_hex(obj_id) + with _write_obj_file(hex_obj_id, self) as f: + for chunk in content_iter: + f.write(chunk) + + return obj_id + def get(self, obj_id): if obj_id not in self: raise ObjNotFoundError(obj_id) @@ -235,6 +247,15 @@ with _read_obj_file(hex_obj_id, self) as f: return f.read() + def get_stream(self, obj_id, bufsize=1024): + if obj_id not in self: + raise ObjNotFoundError(obj_id) + + hex_obj_id = hashutil.hash_to_hex(obj_id) + with _read_obj_file(hex_obj_id, self) as f: + reader = functools.partial(f.read, bufsize) + yield from iter(reader, b'') + def check(self, obj_id): if obj_id not in self: raise ObjNotFoundError(obj_id)