Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/objstorage.py
# Copyright (C) 2015-2018 The Software Heritage developers | # Copyright (C) 2015-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import abc | import abc | ||||
from itertools import dropwhile, islice | from itertools import dropwhile, islice | ||||
import bz2 | import bz2 | ||||
import lzma | import lzma | ||||
import zlib | import zlib | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from typing import Dict | |||||
from .exc import ObjNotFoundError | from .exc import ObjNotFoundError | ||||
ID_HASH_ALGO = 'sha1' | ID_HASH_ALGO = 'sha1' | ||||
ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. | ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. | ||||
DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # Size in bytes of the streaming chunks | DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024 # Size in bytes of the streaming chunks | ||||
DEFAULT_LIMIT = 10000 | DEFAULT_LIMIT = 10000 | ||||
▲ Show 20 Lines • Show All 122 Lines • ▼ Show 20 Lines | def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): | ||||
content should be verified before adding the file. | content should be verified before adding the file. | ||||
Returns: | Returns: | ||||
the id (bytes) of the object into the storage. | the id (bytes) of the object into the storage. | ||||
""" | """ | ||||
pass | pass | ||||
def add_batch(self, contents, check_presence=True): | def add_batch(self, contents, check_presence=True) -> Dict: | ||||
"""Add a batch of new objects to the object storage. | """Add a batch of new objects to the object storage. | ||||
Args: | Args: | ||||
contents (dict): mapping from obj_id to object contents | contents: mapping from obj_id to object contents | ||||
Returns: | Returns: | ||||
the number of objects added to the storage | the summary of objects added to the storage (count of object, | ||||
count of bytes object) | |||||
""" | """ | ||||
ctr = 0 | summary = {'object:add': 0, 'object:add:bytes': 0} | ||||
for obj_id, content in contents.items(): | for obj_id, content in contents.items(): | ||||
self.add(content, obj_id, check_presence=check_presence) | if check_presence and obj_id in self: | ||||
ctr += 1 | continue | ||||
return ctr | self.add(content, obj_id, check_presence=False) | ||||
summary['object:add'] += 1 | |||||
summary['object:add:bytes'] += len(content) | |||||
return summary | |||||
def restore(self, content, obj_id=None, *args, **kwargs): | def restore(self, content, obj_id=None, *args, **kwargs): | ||||
vlorentz: s/content/object/ | |||||
Done Inline Actionsyup, incoming updates. ardumont: yup, incoming updates. | |||||
"""Restore a content that have been corrupted. | """Restore a content that have been corrupted. | ||||
This function is identical to add but does not check if | This function is identical to add but does not check if | ||||
the object id is already in the file system. | the object id is already in the file system. | ||||
The default implementation provided by the current class is | The default implementation provided by the current class is | ||||
suitable for most cases. | suitable for most cases. | ||||
Args: | Args: | ||||
▲ Show 20 Lines • Show All 168 Lines • Show Last 20 Lines |
s/content/object/