diff --git a/swh/objstorage/api/client.py b/swh/objstorage/api/client.py index 8f84336..1a670cb 100644 --- a/swh/objstorage/api/client.py +++ b/swh/objstorage/api/client.py @@ -1,146 +1,94 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - import pickle import requests from requests.exceptions import ConnectionError from ..objstorage import ObjStorage from ..exc import ObjStorageAPIError from .common import (decode_response, encode_data_client as encode_data) class RemoteObjStorage(ObjStorage): """ Proxy to a remote object storage. This class allows to connect to an object storage server via http protocol. Attributes: base_url (string): The url of the server to connect. Must end with a '/' session: The session to send requests. """ def __init__(self, base_url): self.base_url = base_url self.session = requests.Session() def url(self, endpoint): return '%s%s' % (self.base_url, endpoint) def post(self, endpoint, data): try: response = self.session.post( self.url(endpoint), data=encode_data(data), headers={'content-type': 'application/x-msgpack'}, ) except ConnectionError as e: print(str(e)) raise ObjStorageAPIError(e) # XXX: this breaks language-independence and should be # replaced by proper unserialization if response.status_code == 400: raise pickle.loads(decode_response(response)) return decode_response(response) def __contains__(self, obj_id): + """ Indicates if the given object is present in the storage + + See base class [ObjStorage]. + """ return self.post('content/contains', {'obj_id': obj_id}) def add(self, content, obj_id=None, check_presence=True): """ Add a new object to the object storage. - Args: - content: content of the object to be added to the storage. - obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When - given, obj_id will be trusted to match the bytes. If missing, - obj_id will be computed on the fly. - check_presence: indicate if the presence of the content should be - verified before adding the file. - - Returns: - the id of the object into the storage. + See base class [ObjStorage]. """ return self.post('content/add', {'bytes': content, 'obj_id': obj_id, 'check_presence': check_presence}) - def restore(self, content, obj_id=None): - """ Restore a content that have been corrupted. - - This function is identical to add_bytes but does not check if - the object id is already in the file system. - - Args: - content: content of the object to be added to the storage - obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When - given, obj_id will be trusted to match bytes. If missing, - obj_id will be computed on the fly. - """ - return self.add(content, obj_id, check_presence=False) - def get(self, obj_id): """ Retrieve the content of a given object. - Args: - obj_id: object id. - - Returns: - the content of the requested object as bytes. - - Raises: - ObjNotFoundError: if the requested object is missing. + See base class [ObjStorage]. """ return self.post('content/get', {'obj_id': obj_id}) def get_batch(self, obj_ids): """ Retrieve content in bulk. - Note: This function does have a default implementation in ObjStorage - that is suitable for most cases. - - Args: - obj_ids: list of object ids. - - Returns: - list of resulting contents, or None if the content could not - be retrieved. Do not raise any exception as a fail for one content - will not cancel the whole request. + See base class [ObjStorage]. """ return self.post('content/get/batch', {'obj_ids': obj_ids}) def check(self, obj_id): """ Perform an integrity check for a given object. - Verify that the file object is in place and that the gziped content - matches the object id. - - Args: - obj_id: object id. - - Raises: - ObjNotFoundError: if the requested object is missing. - Error: if the request object is corrupted. + See base class [ObjStorage]. """ self.post('content/check', {'obj_id': obj_id}) def get_random(self, batch_size): """ Get random ids of existing contents - This method is used in order to get random ids to perform - content integrity verifications on random contents. - - Attributes: - batch_size (int): Number of ids that will be given - - Yields: - An iterable of ids of contents that are in the current object - storage. + See base class [ObjStorage]. """ return self.post('content/get/random', {'batch_size': batch_size}) diff --git a/swh/objstorage/multiplexer/filter/filter.py b/swh/objstorage/multiplexer/filter/filter.py index fa4cc06..747c7fc 100644 --- a/swh/objstorage/multiplexer/filter/filter.py +++ b/swh/objstorage/multiplexer/filter/filter.py @@ -1,48 +1,86 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from ...objstorage import ObjStorage class ObjStorageFilter(ObjStorage): """ Base implementation of a filter that allow inputs on ObjStorage or not This class copy the API of ...objstorage in order to filter the inputs of this class. If the operation is allowed, return the result of this operation applied to the destination implementation. Otherwise, just return without any operation. This class is an abstract base class for a classic read/write storage. Filters can inherit from it and only redefine some methods in order to change behavior. """ def __init__(self, storage): self.storage = storage def __contains__(self, *args, **kwargs): + """ Indicates if the given object is present in the storage + + See base class [ObjStorage]. + """ return self.storage.__contains__(*args, **kwargs) def __iter__(self): + """ Iterates over the content of each storages + + Warning: The `__iter__` methods frequently have bad performance. You + almost certainly don't want to use this method in production as the + wrapped storage may cause performance issues. + """ return self.storage.__iter__() def __len__(self): + """ Compute the number of objects in the current object storage. + + Warning: performance issue in `__iter__` also applies here. + + Returns: + number of objects contained in the storage. + """ return self.storage.__len__() - def add(self, *args, **kwargs): - return self.storage.add(*args, **kwargs) + def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): + """ Add a new object to the object storage. + + See base class [ObjStorage]. + """ + return self.storage.add(content, obj_id, check_presence, + *args, **kwargs) + + def restore(self, content, obj_id=None, *args, **kwargs): + """ Restore a content that have been corrupted. + + See base class [ObjStorage] & self.add() method. + """ + return self.storage.restore(content, obj_id, *args, **kwargs) + + def get(self, obj_id, *args, **kwargs): + """ Retrieve the content of a given object. + + See base class [ObjStorage]. + """ + return self.storage.get(obj_id, *args, **kwargs) - def restore(self, *args, **kwargs): - return self.storage.restore(*args, **kwargs) + def check(self, obj_id, *args, **kwargs): + """ Perform an integrity check for a given object. - def get(self, *args, **kwargs): - return self.storage.get(*args, **kwargs) + See base class [ObjStorage]. + """ + return self.storage.check(obj_id, *args, **kwargs) - def check(self, *args, **kwargs): - return self.storage.check(*args, **kwargs) + def get_random(self, batch_size, *args, **kwargs): + """ Get random ids of existing contents - def get_random(self, *args, **kwargs): - return self.storage.get_random(*args, **kwargs) + See base class [ObjStorage]. + """ + return self.storage.get_random(batch_size, *args, **kwargs) diff --git a/swh/objstorage/multiplexer/filter/id_filter.py b/swh/objstorage/multiplexer/filter/id_filter.py index 71039b0..ca149b9 100644 --- a/swh/objstorage/multiplexer/filter/id_filter.py +++ b/swh/objstorage/multiplexer/filter/id_filter.py @@ -1,99 +1,117 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import re from swh.core import hashutil from .filter import ObjStorageFilter from ...objstorage import ID_HASH_ALGO from ...exc import ObjNotFoundError def compute_hash(bytes): """ Compute the hash of the given content. """ # Checksum is missing, compute it on the fly. h = hashutil._new_hash(ID_HASH_ALGO, len(bytes)) h.update(bytes) return h.digest() class IdObjStorageFilter(ObjStorageFilter): """ Filter that only allow operations if the object id match a requirement. Even for read operations, check before if the id match the requirements. This may prevent for unnecesary disk access. """ def is_valid(self, obj_id): """ Indicates if the given id is valid. """ raise NotImplementedError('Implementations of an IdObjStorageFilter ' 'must have a "is_valid" method') def __contains__(self, obj_id, *args, **kwargs): + """ Indicates if the given object is present in the storage + + See base class [ObjStorage]. + """ if self.is_valid(obj_id): return self.storage.__contains__(*args, obj_id=obj_id, **kwargs) return False def __len__(self): + """ See base class [ObjStorageFilter]. + """ return sum(1 for i in [id for id in self.storage if self.is_valid(id)]) def __iter__(self): + """ See base class [ObjStorageFilter]. + """ yield from filter(lambda id: self.is_valid(id), iter(self.storage)) def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): + """ See base class [ObjStorageFilter]. + """ if obj_id is None: obj_id = compute_hash(content) if self.is_valid(obj_id): return self.storage.add(content, *args, obj_id=obj_id, **kwargs) def restore(self, content, obj_id=None, *args, **kwargs): + """ See base class [ObjStorageFilter]. + """ if obj_id is None: obj_id = compute_hash(content) if self.is_valid(obj_id): return self.storage.restore(content, *args, obj_id=obj_id, **kwargs) def get(self, obj_id, *args, **kwargs): + """ See base class [ObjStorageFilter]. + """ if self.is_valid(obj_id): return self.storage.get(*args, obj_id=obj_id, **kwargs) raise ObjNotFoundError(obj_id) def check(self, obj_id, *args, **kwargs): + """ See base class [ObjStorageFilter]. + """ if self.is_valid(obj_id): return self.storage.check(*args, obj_id=obj_id, **kwargs) raise ObjNotFoundError(obj_id) def get_random(self, *args, **kwargs): + """ See base class [ObjStorageFilter]. + """ yield from filter(lambda id: self.is_valid(id), self.storage.get_random(*args, **kwargs)) class RegexIdObjStorageFilter(IdObjStorageFilter): """ Filter that allow operations if the content's id as hex match a regex. """ def __init__(self, storage, regex): super().__init__(storage) self.regex = re.compile(regex) def is_valid(self, obj_id): hex_obj_id = hashutil.hash_to_hex(obj_id) return self.regex.match(hex_obj_id) is not None class PrefixIdObjStorageFilter(IdObjStorageFilter): """ Filter that allow operations if the hexlified id have a given prefix. """ def __init__(self, storage, prefix): super().__init__(storage) self.prefix = str(prefix) def is_valid(self, obj_id): hex_obj_id = hashutil.hash_to_hex(obj_id) return str(hex_obj_id).startswith(self.prefix) diff --git a/swh/objstorage/multiplexer/filter/read_write_filter.py b/swh/objstorage/multiplexer/filter/read_write_filter.py index e4821b9..541bcc5 100644 --- a/swh/objstorage/multiplexer/filter/read_write_filter.py +++ b/swh/objstorage/multiplexer/filter/read_write_filter.py @@ -1,17 +1,16 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from .filter import ObjStorageFilter class ReadObjStorageFilter(ObjStorageFilter): """ Filter that disable write operation of the storage. """ - def add(self, *args, **kwargs): return def restore(self, *args, **kwargs): return diff --git a/swh/objstorage/multiplexer/multiplexer_objstorage.py b/swh/objstorage/multiplexer/multiplexer_objstorage.py index 9376498..f7a355e 100644 --- a/swh/objstorage/multiplexer/multiplexer_objstorage.py +++ b/swh/objstorage/multiplexer/multiplexer_objstorage.py @@ -1,194 +1,174 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import random from ..objstorage import ObjStorage from ..exc import ObjNotFoundError class MultiplexerObjStorage(ObjStorage): """ Implementation of ObjStorage that distribute between multiple storages The multiplexer object storage allows an input to be demultiplexed among multiple storages that will or will not accept it by themselves (see .filter package). As the ids can be differents, no pre-computed ids should be submitted. Also, there are no guarantees that the returned ids can be used directly into the storages that the multiplexer manage. Use case examples could be: Example 1: storage_v1 = filter.read_only(PathSlicingObjStorage('/dir1', '0:2/2:4/4:6')) storage_v2 = PathSlicingObjStorage('/dir2', '0:1/0:5') storage = MultiplexerObjStorage([storage_v1, storage_v2]) When using 'storage', all the new contents will only be added to the v2 storage, while it will be retrievable from both. Example 2: storage_v1 = filter.id_regex( PathSlicingObjStorage('/dir1', '0:2/2:4/4:6'), r'[^012].*' ) storage_v2 = filter.if_regex( PathSlicingObjStorage('/dir2', '0:1/0:5'), r'[012]/*' ) storage = MultiplexerObjStorage([storage_v1, storage_v2]) When using this storage, the contents with a sha1 starting with 0, 1 or 2 will be redirected (read AND write) to the storage_v2, while the others will be redirected to the storage_v1. If a content starting with 0, 1 or 2 is present in the storage_v1, it would be ignored anyway. """ def __init__(self, storages): self.storages = storages def __contains__(self, obj_id): + """ Indicates if the given object is present in the storage + + See base class [ObjStorage]. + """ for storage in self.storages: if obj_id in storage: return True return False def __iter__(self): + """ Iterates over the content of each storages + + Due to the demultiplexer nature, same content can be in multiple + storages and may be yielded multiple times. + + Warning: The `__iter__` methods frequently have bad performance. You + almost certainly don't want to use this method in production. + """ for storage in self.storages: yield from storage def __len__(self): - """ Returns the number of files in the storage. + """ Compute the number of objects in the current object storage. + + Identical objects present in multiple storages will be counted as + multiple objects. + Warning: this currently uses `__iter__`, its warning about bad + performance applies. - Warning: Multiple files may represent the same content, so this method - does not indicate how many different contents are in the storage. + Returns: + number of objects contained in the storage. """ return sum(map(len, self.storages)) def add(self, content, obj_id=None, check_presence=True): """ Add a new object to the object storage. If the adding step works in all the storages that accept this content, this is a success. Otherwise, the full adding step is an error even if it succeed in some of the storages. Args: content: content of the object to be added to the storage. obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When given, obj_id will be trusted to match the bytes. If missing, obj_id will be computed on the fly. check_presence: indicate if the presence of the content should be verified before adding the file. Returns: an id of the object into the storage. As the write-storages are always readable as well, any id will be valid to retrieve a content. """ return [storage.add(content, obj_id, check_presence) for storage in self.storages].pop() def restore(self, content, obj_id=None): """ Restore a content that have been corrupted. - This function is identical to add_bytes but does not check if - the object id is already in the file system. - - (see "add" method) - - Args: - content: content of the object to be added to the storage - obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When - given, obj_id will be trusted to match bytes. If missing, - obj_id will be computed on the fly. - - Returns: - an id of the object into the storage. As the write-storages are - always readable as well, any id will be valid to retrieve a - content. + See base class [ObjStorage] & self.add() method. """ return [storage.restore(content, obj_id) for storage in self.storages].pop() def get(self, obj_id): """ Retrieve the content of a given object. - Args: - obj_id: object id. - - Returns: - the content of the requested object as bytes. - - Raises: - ObjNotFoundError: if the requested object is missing. + See base class [ObjStorage]. """ for storage in self.storages: try: return storage.get(obj_id) except ObjNotFoundError: continue # If no storage contains this content, raise the error raise ObjNotFoundError(obj_id) def check(self, obj_id): """ Perform an integrity check for a given object. - Verify that the file object is in place and that the gziped content - matches the object id. - - Args: - obj_id: object id. - - Raises: - ObjNotFoundError: if the requested object is missing. - Error: if the request object is corrupted. + See base class [ObjStorage]. """ nb_present = 0 for storage in self.storages: try: storage.check(obj_id) except ObjNotFoundError: continue else: nb_present += 1 # If there is an Error because of a corrupted file, then let it pass. # Raise the ObjNotFoundError only if the content coulnd't be found in # all the storages. if nb_present == 0: raise ObjNotFoundError(obj_id) def get_random(self, batch_size): """ Get random ids of existing contents - This method is used in order to get random ids to perform - content integrity verifications on random contents. - - Attributes: - batch_size (int): Number of ids that will be given - - Yields: - An iterable of ids of contents that are in the current object - storage. + See base class [ObjStorage]. """ storages_set = [storage for storage in self.storages if len(storage) > 0] if len(storages_set) <= 0: return [] while storages_set: storage = random.choice(storages_set) try: return storage.get_random(batch_size) except NotImplementedError: storages_set.remove(storage) # There is no storage that allow the get_random operation raise NotImplementedError( "There is no storage implementation into the multiplexer that " "support the 'get_random' operation" ) diff --git a/swh/objstorage/objstorage.py b/swh/objstorage/objstorage.py index 1e2dcea..e9883b9 100644 --- a/swh/objstorage/objstorage.py +++ b/swh/objstorage/objstorage.py @@ -1,141 +1,151 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from .exc import ObjNotFoundError ID_HASH_ALGO = 'sha1' ID_HASH_LENGTH = 40 # Size in bytes of the hash hexadecimal representation. class ObjStorage(): """ High-level API to manipulate the Software Heritage object storage. Conceptually, the object storage offers 5 methods: - __contains__() check if an object is present, by object id - add() add a new object, returning an object id - restore() same as add() but erase an already existed content - get() retrieve the content of an object, by object id - check() check the integrity of an object, by object id And some management methods: - get_random() get random object id of existing contents (used for the content integrity checker). Each implementation of this interface can have a different behavior and its own way to store the contents. """ def __contains__(self, *args, **kwargs): + def __contains__(self, obj_id, *args, **kwargs): + """ Indicates if the given object is present in the storage + + Returns: + True iff the object is present in the current object storage. + """ raise NotImplementedError( "Implementations of ObjStorage must have a '__contains__' method" ) def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): """ Add a new object to the object storage. Args: content: content of the object to be added to the storage. obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When given, obj_id will be trusted to match the bytes. If missing, obj_id will be computed on the fly. check_presence: indicate if the presence of the content should be verified before adding the file. Returns: the id of the object into the storage. """ raise NotImplementedError( "Implementations of ObjStorage must have a 'add' method" ) def restore(self, content, obj_id=None, *args, **kwargs): """ Restore a content that have been corrupted. This function is identical to add_bytes but does not check if the object id is already in the file system. + The default implementation provided by the current class is + suitable for most cases. Args: content: content of the object to be added to the storage obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When given, obj_id will be trusted to match bytes. If missing, obj_id will be computed on the fly. """ - raise NotImplemented( - "Implementations of ObjStorage must have a 'restore' method" - ) + # check_presence to false will erase the potential previous content. + return self.add(content, obj_id, check_presence=False) def get(self, obj_id, *args, **kwargs): """ Retrieve the content of a given object. Args: obj_id: object id. Returns: the content of the requested object as bytes. Raises: ObjNotFoundError: if the requested object is missing. """ raise NotImplementedError( "Implementations of ObjStorage must have a 'get' method" ) def get_batch(self, obj_ids, *args, **kwargs): """ Retrieve content in bulk. Note: This function does have a default implementation in ObjStorage that is suitable for most cases. + For object storages that needs to do the minimal number of requests + possible (ex: remote object storages), that method can be overriden + to perform a more efficient operation. Args: obj_ids: list of object ids. Returns: list of resulting contents, or None if the content could not be retrieved. Do not raise any exception as a fail for one content will not cancel the whole request. """ for obj_id in obj_ids: try: yield self.get(obj_id) except ObjNotFoundError: yield None def check(self, obj_id, *args, **kwargs): """ Perform an integrity check for a given object. Verify that the file object is in place and that the gziped content matches the object id. Args: obj_id: object id. Raises: ObjNotFoundError: if the requested object is missing. Error: if the request object is corrupted. """ raise NotImplementedError( "Implementations of ObjStorage must have a 'check' method" ) def get_random(self, batch_size, *args, **kwargs): """ Get random ids of existing contents This method is used in order to get random ids to perform content integrity verifications on random contents. - Attributes: + Args: batch_size (int): Number of ids that will be given Yields: An iterable of ids of contents that are in the current object storage. """ raise NotImplementedError( "The current implementation of ObjStorage does not support " "'get_random' operation" ) diff --git a/swh/objstorage/objstorage_pathslicing.py b/swh/objstorage/objstorage_pathslicing.py index 1ea2ebe..a855d70 100644 --- a/swh/objstorage/objstorage_pathslicing.py +++ b/swh/objstorage/objstorage_pathslicing.py @@ -1,347 +1,299 @@ # Copyright (C) 2015-2016 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import gzip import tempfile import random from contextlib import contextmanager from swh.core import hashutil from .objstorage import ObjStorage, ID_HASH_ALGO, ID_HASH_LENGTH from .exc import ObjNotFoundError, Error GZIP_BUFSIZ = 1048576 DIR_MODE = 0o755 FILE_MODE = 0o644 @contextmanager def _write_obj_file(hex_obj_id, objstorage): """ Context manager for writing object files to the object storage. During writing, data are written to a temporary file, which is atomically renamed to the right file name after closing. This context manager also takes care of (gzip) compressing the data on the fly. Usage sample: with _write_obj_file(hex_obj_id, objstorage): f.write(obj_data) Yields: a file-like object open for writing bytes. """ # Get the final paths and create the directory if absent. dir = objstorage._obj_dir(hex_obj_id) if not os.path.isdir(dir): os.makedirs(dir, DIR_MODE, exist_ok=True) path = os.path.join(dir, hex_obj_id) # Create a temporary file. (tmp, tmp_path) = tempfile.mkstemp(suffix='.tmp', prefix='hex_obj_id.', dir=dir) # Open the file and yield it for writing. tmp_f = os.fdopen(tmp, 'wb') with gzip.GzipFile(filename=tmp_path, fileobj=tmp_f) as f: yield f # Then close the temporary file and move it to the right directory. tmp_f.close() os.chmod(tmp_path, FILE_MODE) os.rename(tmp_path, path) @contextmanager def _read_obj_file(hex_obj_id, objstorage): """ Context manager for reading object file in the object storage. Usage sample: with _read_obj_file(hex_obj_id, objstorage) as f: b = f.read() Yields: a file-like object open for reading bytes. """ path = objstorage._obj_path(hex_obj_id) with gzip.GzipFile(path, 'rb') as f: yield f class PathSlicingObjStorage(ObjStorage): """ Implementation of the ObjStorage API based on the hash of the content. On disk, an object storage is a directory tree containing files named after their object IDs. An object ID is a checksum of its content, depending on the value of the ID_HASH_ALGO constant (see hashutil for its meaning). To avoid directories that contain too many files, the object storage has a given slicing. Each slicing correspond to a directory that is named according to the hash of its content. So for instance a file with SHA1 34973274ccef6ab4dfaaf86599792fa9c3fe4689 will be stored in the given object storages : - 0:2/2:4/4:6 : 34/97/32/34973274ccef6ab4dfaaf86599792fa9c3fe4689 - 0:1/0:5/ : 3/34973/34973274ccef6ab4dfaaf86599792fa9c3fe4689 The files in the storage are stored in gzipped compressed format. Attributes: root (string): path to the root directory of the storage on the disk. bounds: list of tuples that indicates the beginning and the end of each subdirectory for a content. """ def __init__(self, root, slicing): """ Create an object to access a hash-slicing based object storage. Args: root (string): path to the root directory of the storage on the disk. slicing (string): string that indicates the slicing to perform on the hash of the content to know the path where it should be stored. """ if not os.path.isdir(root): raise ValueError( 'PathSlicingObjStorage root "%s" is not a directory' % root ) self.root = root # Make a list of tuples where each tuple contains the beginning # and the end of each slicing. self.bounds = [ slice(*map(int, sbounds.split(':'))) for sbounds in slicing.split('/') if sbounds ] max_endchar = max(map(lambda bound: bound.stop, self.bounds)) if ID_HASH_LENGTH < max_endchar: raise ValueError( 'Algorithm %s has too short hash for slicing to char %d' % (ID_HASH_ALGO, max_endchar) ) def __contains__(self, obj_id): - """ Check whether the given object is present in the storage or not. + """ Indicates if the given object is present in the storage - Returns: - True iff the object is present in the storage. + See base class [ObjStorage]. """ hex_obj_id = hashutil.hash_to_hex(obj_id) return os.path.exists(self._obj_path(hex_obj_id)) def __iter__(self): """iterate over the object identifiers currently available in the storage Warning: with the current implementation of the object storage, this method will walk the filesystem to list objects, meaning that listing all objects will be very slow for large storages. You almost certainly don't want to use this method in production. Return: iterator over object IDs """ def obj_iterator(): # XXX hackish: it does not verify that the depth of found files # matches the slicing depth of the storage for root, _dirs, files in os.walk(self.root): for f in files: yield bytes.fromhex(f) return obj_iterator() def __len__(self): """compute the number of objects available in the storage Warning: this currently uses `__iter__`, its warning about bad performances applies Return: number of objects contained in the storage - """ return sum(1 for i in self) def _obj_dir(self, hex_obj_id): """ Compute the storage directory of an object. See also: PathSlicingObjStorage::_obj_path Args: hex_obj_id: object id as hexlified string. Returns: Path to the directory that contains the required object. """ slices = [hex_obj_id[bound] for bound in self.bounds] return os.path.join(self.root, *slices) def _obj_path(self, hex_obj_id): """ Compute the full path to an object into the current storage. See also: PathSlicingObjStorage::_obj_dir Args: hex_obj_id: object id as hexlified string. Returns: Path to the actual object corresponding to the given id. """ return os.path.join(self._obj_dir(hex_obj_id), hex_obj_id) def add(self, bytes, obj_id=None, check_presence=True): - """ Add a new object to the object storage. + """ Add a new object to the current object storage. - Args: - bytes: content of the object to be added to the storage. - obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When - given, obj_id will be trusted to match the bytes. If missing, - obj_id will be computed on the fly. - check_presence: indicate if the presence of the content should be - verified before adding the file. - - Returns: - the id of the object into the storage. + See base class [ObjStorage]. """ if obj_id is None: # Checksum is missing, compute it on the fly. h = hashutil._new_hash(ID_HASH_ALGO, len(bytes)) h.update(bytes) obj_id = h.digest() if check_presence and obj_id in self: # If the object is already present, return immediatly. return obj_id hex_obj_id = hashutil.hash_to_hex(obj_id) with _write_obj_file(hex_obj_id, self) as f: f.write(bytes) return obj_id - def restore(self, bytes, obj_id=None): - """ Restore a content that have been corrupted. - - This function is identical to add_bytes but does not check if - the object id is already in the file system. - - Args: - bytes: content of the object to be added to the storage - obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When - given, obj_id will be trusted to match bytes. If missing, - obj_id will be computed on the fly. - """ - return self.add(bytes, obj_id, check_presence=False) - def get(self, obj_id): """ Retrieve the content of a given object. - Args: - obj_id: object id. - - Returns: - the content of the requested object as bytes. - - Raises: - ObjNotFoundError: if the requested object is missing. + See base class [ObjStorage]. """ if obj_id not in self: raise ObjNotFoundError(obj_id) # Open the file and return its content as bytes hex_obj_id = hashutil.hash_to_hex(obj_id) with _read_obj_file(hex_obj_id, self) as f: return f.read() def check(self, obj_id): """ Perform an integrity check for a given object. - Verify that the file object is in place and that the gziped content - matches the object id. - - Args: - obj_id: object id. - - Raises: - ObjNotFoundError: if the requested object is missing. - Error: if the request object is corrupted. + See base class [ObjStorage]. """ if obj_id not in self: raise ObjNotFoundError(obj_id) hex_obj_id = hashutil.hash_to_hex(obj_id) try: with gzip.open(self._obj_path(hex_obj_id)) as f: length = None if ID_HASH_ALGO.endswith('_git'): # if the hashing algorithm is git-like, we need to know the # content size to hash on the fly. Do a first pass here to # compute the size length = 0 while True: chunk = f.read(GZIP_BUFSIZ) length += len(chunk) if not chunk: break f.rewind() checksums = hashutil._hash_file_obj(f, length, algorithms=[ID_HASH_ALGO]) actual_obj_id = checksums[ID_HASH_ALGO] if obj_id != actual_obj_id: raise Error( 'Corrupt object %s should have id %s' % (hashutil.hash_to_hex(obj_id), hashutil.hash_to_hex(actual_obj_id)) ) except (OSError, IOError): # IOError is for compatibility with older python versions raise Error('Corrupt object %s is not a gzip file' % obj_id) def get_random(self, batch_size): """ Get random ids of existing contents - This method is used in order to get random ids to perform - content integrity verifications on random contents. - - Attributes: - batch_size (int): Number of ids that will be given - - Yields: - An iterable of ids of contents that are in the current object - storage. + See base class [ObjStorage]. """ def get_random_content(self, batch_size): """ Get a batch of content inside a single directory. Returns: a tuple (batch size, batch). """ dirs = [] for level in range(len(self.bounds)): path = os.path.join(self.root, *dirs) dir_list = next(os.walk(path))[1] if 'tmp' in dir_list: dir_list.remove('tmp') dirs.append(random.choice(dir_list)) path = os.path.join(self.root, *dirs) content_list = next(os.walk(path))[2] length = min(batch_size, len(content_list)) return length, map(hashutil.hex_to_hash, random.sample(content_list, length)) while batch_size: length, it = get_random_content(self, batch_size) batch_size = batch_size - length yield from it