Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/objstorage.py
| # Copyright (C) 2015-2018 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import abc | import abc | ||||
| from itertools import dropwhile, islice | from itertools import dropwhile, islice | ||||
| import bz2 | import bz2 | ||||
| import lzma | import lzma | ||||
| import zlib | import zlib | ||||
| from swh.model import hashutil | from swh.model import hashutil | ||||
| from typing import Dict | |||||
| from .exc import ObjNotFoundError | from .exc import ObjNotFoundError | ||||
| ID_HASH_ALGO = 'sha1' | ID_HASH_ALGO = 'sha1' | ||||
| ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. | ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. | ||||
| DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # Size in bytes of the streaming chunks | DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # Size in bytes of the streaming chunks | ||||
| DEFAULT_LIMIT = 10000 | DEFAULT_LIMIT = 10000 | ||||
| ▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): | ||||
| content should be verified before adding the file. | content should be verified before adding the file. | ||||
| Returns: | Returns: | ||||
| the id (bytes) of the object into the storage. | the id (bytes) of the object into the storage. | ||||
| """ | """ | ||||
| pass | pass | ||||
| def add_batch(self, contents, check_presence=True): | def add_batch(self, contents, check_presence=True) -> Dict: | ||||
| """Add a batch of new objects to the object storage. | """Add a batch of new objects to the object storage. | ||||
| Args: | Args: | ||||
| contents (dict): mapping from obj_id to object contents | contents: mapping from obj_id to object contents | ||||
| Returns: | Returns: | ||||
| the number of objects added to the storage | the summary of objects added to the storage (count of object, | ||||
| count of bytes object) | |||||
| """ | """ | ||||
| ctr = 0 | object_add = 0 | ||||
| object_add_bytes = 0 | |||||
| for obj_id, content in contents.items(): | for obj_id, content in contents.items(): | ||||
| self.add(content, obj_id, check_presence=check_presence) | if check_presence and obj_id in self: | ||||
| ctr += 1 | continue | ||||
| return ctr | self.add(content, obj_id, check_presence=False) | ||||
| object_add += 1 | |||||
| object_add_bytes += len(content) | |||||
| return { | |||||
| 'object:add': object_add, | |||||
vlorentz: s/content/object/ | |||||
Done Inline Actionsyup, incoming updates. ardumont: yup, incoming updates. | |||||
| 'object:add:bytes': object_add_bytes, | |||||
| } | |||||
| def restore(self, content, obj_id=None, *args, **kwargs): | def restore(self, content, obj_id=None, *args, **kwargs): | ||||
| """Restore a content that have been corrupted. | """Restore a content that have been corrupted. | ||||
| This function is identical to add but does not check if | This function is identical to add but does not check if | ||||
| the object id is already in the file system. | the object id is already in the file system. | ||||
| The default implementation provided by the current class is | The default implementation provided by the current class is | ||||
| suitable for most cases. | suitable for most cases. | ||||
| ▲ Show 20 Lines • Show All 170 Lines • Show Last 20 Lines | |||||
s/content/object/