Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/multiplexer/filter/id_filter.py
- This file was added.
| # Copyright (C) 2015-2016 The Software Heritage developers | |||||
| # See the AUTHORS file at the top-level directory of this distribution | |||||
| # License: GNU General Public License version 3, or any later version | |||||
| # See top-level LICENSE file for more information | |||||
| import re | |||||
| from swh.core import hashutil | |||||
| from .filter import ObjStorageFilter | |||||
| from ...objstorage import ID_HASH_ALGO | |||||
| from ...exc import ObjNotFoundError | |||||
| def compute_hash(bytes): | |||||
| """ Compute the hash of the given content. | |||||
| """ | |||||
| # Checksum is missing, compute it on the fly. | |||||
| h = hashutil._new_hash(ID_HASH_ALGO, len(bytes)) | |||||
| h.update(bytes) | |||||
| return h.digest() | |||||
| class IdObjStorageFilter(ObjStorageFilter): | |||||
| """ Filter that only allow operations if the object id match a requirement. | |||||
| Even for read operations, check before if the id match the requirements. | |||||
| This may prevent for unnecesary disk access. | |||||
| """ | |||||
| def is_valid(self, obj_id): | |||||
| """ Indicates if the given id is valid. | |||||
| """ | |||||
| raise NotImplementedError('Implementations of an IdObjStorageFilter ' | |||||
| 'must have a "is_valid" method') | |||||
| def __contains__(self, obj_id, *args, **kwargs): | |||||
| if self.is_valid(obj_id): | |||||
| return self.storage.__contains__(*args, obj_id=obj_id, **kwargs) | |||||
| return False | |||||
| def __len__(self): | |||||
| return sum(1 for i in [id for id in self.storage if self.is_valid(id)]) | |||||
| def __iter__(self): | |||||
| yield from filter(lambda id: self.is_valid(id), iter(self.storage)) | |||||
| def add(self, content, obj_id=None, check_presence=True, *args, **kwargs): | |||||
| if obj_id is None: | |||||
| obj_id = compute_hash(content) | |||||
| if self.is_valid(obj_id): | |||||
| return self.storage.add(content, *args, obj_id=obj_id, **kwargs) | |||||
| def restore(self, content, obj_id=None, *args, **kwargs): | |||||
| if obj_id is None: | |||||
| obj_id = compute_hash(content) | |||||
| if self.is_valid(obj_id): | |||||
| return self.storage.restore(content, *args, | |||||
| obj_id=obj_id, **kwargs) | |||||
| def get(self, obj_id, *args, **kwargs): | |||||
| if self.is_valid(obj_id): | |||||
| return self.storage.get(*args, obj_id=obj_id, **kwargs) | |||||
| raise ObjNotFoundError(obj_id) | |||||
| def check(self, obj_id, *args, **kwargs): | |||||
| if self.is_valid(obj_id): | |||||
| return self.storage.check(*args, obj_id=obj_id, **kwargs) | |||||
| raise ObjNotFoundError(obj_id) | |||||
| def get_random(self, *args, **kwargs): | |||||
| yield from filter(lambda id: self.is_valid(id), | |||||
| self.storage.get_random(*args, **kwargs)) | |||||
| class RegexIdObjStorageFilter(IdObjStorageFilter): | |||||
| """ Filter that allow operations if the content's id as hex match a regex. | |||||
| """ | |||||
| def __init__(self, storage, regex): | |||||
| super().__init__(storage) | |||||
| self.regex = re.compile(regex) | |||||
| def is_valid(self, obj_id): | |||||
| hex_obj_id = hashutil.hash_to_hex(obj_id) | |||||
| return self.regex.match(hex_obj_id) is not None | |||||
| class PrefixIdObjStorageFilter(IdObjStorageFilter): | |||||
| """ Filter that allow operations if the hexlified id have a given prefix. | |||||
| """ | |||||
| def __init__(self, storage, prefix): | |||||
| super().__init__(storage) | |||||
| self.prefix = str(prefix) | |||||
| def is_valid(self, obj_id): | |||||
| hex_obj_id = hashutil.hash_to_hex(obj_id) | |||||
| return str(hex_obj_id).startswith(self.prefix) | |||||