Changeset View
Changeset View
Standalone View
Standalone View
swh/objstorage/api/client.py
# Copyright (C) 2015 The Software Heritage developers | # Copyright (C) 2015-2016 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import pickle | import pickle | ||||
import requests | import requests | ||||
from requests.exceptions import ConnectionError | from requests.exceptions import ConnectionError | ||||
from ..objstorage import ObjStorage | from ..objstorage import ObjStorage | ||||
from ..exc import ObjStorageAPIError | from ..exc import ObjStorageAPIError | ||||
from .common import (decode_response, | from .common import (decode_response, | ||||
Show All 32 Lines | def post(self, endpoint, data): | ||||
# XXX: this breaks language-independence and should be | # XXX: this breaks language-independence and should be | ||||
# replaced by proper unserialization | # replaced by proper unserialization | ||||
if response.status_code == 400: | if response.status_code == 400: | ||||
raise pickle.loads(decode_response(response)) | raise pickle.loads(decode_response(response)) | ||||
return decode_response(response) | return decode_response(response) | ||||
def __contains__(self, obj_id): | def __contains__(self, obj_id): | ||||
""" Indicates if the given object is present in the storage | |||||
See base class [ObjStorage]. | |||||
""" | |||||
return self.post('content/contains', {'obj_id': obj_id}) | return self.post('content/contains', {'obj_id': obj_id}) | ||||
def add(self, content, obj_id=None, check_presence=True): | def add(self, content, obj_id=None, check_presence=True): | ||||
""" Add a new object to the object storage. | """ Add a new object to the object storage. | ||||
Args: | See base class [ObjStorage]. | ||||
content: content of the object to be added to the storage. | |||||
obj_id: checksum of [bytes] using [ID_HASH_ALGO] algorithm. When | |||||
given, obj_id will be trusted to match the bytes. If missing, | |||||
obj_id will be computed on the fly. | |||||
check_presence: indicate if the presence of the content should be | |||||
verified before adding the file. | |||||
Returns: | |||||
the id of the object into the storage. | |||||
""" | """ | ||||
return self.post('content/add', {'bytes': content, 'obj_id': obj_id, | return self.post('content/add', {'bytes': content, 'obj_id': obj_id, | ||||
'check_presence': check_presence}) | 'check_presence': check_presence}) | ||||
def restore(self, content, obj_id=None): | |||||
""" Restore a content that have been corrupted. | |||||
This function is identical to add_bytes but does not check if | |||||
the object id is already in the file system. | |||||
Args: | |||||
content: content of the object to be added to the storage | |||||
obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When | |||||
given, obj_id will be trusted to match bytes. If missing, | |||||
obj_id will be computed on the fly. | |||||
""" | |||||
return self.add(content, obj_id, check_presence=False) | |||||
def get(self, obj_id): | def get(self, obj_id): | ||||
""" Retrieve the content of a given object. | """ Retrieve the content of a given object. | ||||
Args: | See base class [ObjStorage]. | ||||
obj_id: object id. | |||||
Returns: | |||||
the content of the requested object as bytes. | |||||
Raises: | |||||
ObjNotFoundError: if the requested object is missing. | |||||
""" | """ | ||||
return self.post('content/get', {'obj_id': obj_id}) | return self.post('content/get', {'obj_id': obj_id}) | ||||
def get_batch(self, obj_ids): | def get_batch(self, obj_ids): | ||||
""" Retrieve content in bulk. | """ Retrieve content in bulk. | ||||
Note: This function does have a default implementation in ObjStorage | See base class [ObjStorage]. | ||||
that is suitable for most cases. | |||||
Args: | |||||
obj_ids: list of object ids. | |||||
Returns: | |||||
list of resulting contents, or None if the content could not | |||||
be retrieved. Do not raise any exception as a fail for one content | |||||
will not cancel the whole request. | |||||
""" | """ | ||||
return self.post('content/get/batch', {'obj_ids': obj_ids}) | return self.post('content/get/batch', {'obj_ids': obj_ids}) | ||||
def check(self, obj_id): | def check(self, obj_id): | ||||
""" Perform an integrity check for a given object. | """ Perform an integrity check for a given object. | ||||
Verify that the file object is in place and that the gziped content | See base class [ObjStorage]. | ||||
matches the object id. | |||||
Args: | |||||
obj_id: object id. | |||||
Raises: | |||||
ObjNotFoundError: if the requested object is missing. | |||||
Error: if the request object is corrupted. | |||||
""" | """ | ||||
self.post('content/check', {'obj_id': obj_id}) | self.post('content/check', {'obj_id': obj_id}) | ||||
def get_random(self, batch_size): | def get_random(self, batch_size): | ||||
""" Get random ids of existing contents | """ Get random ids of existing contents | ||||
This method is used in order to get random ids to perform | See base class [ObjStorage]. | ||||
content integrity verifications on random contents. | |||||
Attributes: | |||||
batch_size (int): Number of ids that will be given | |||||
Yields: | |||||
An iterable of ids of contents that are in the current object | |||||
storage. | |||||
""" | """ | ||||
return self.post('content/get/random', {'batch_size': batch_size}) | return self.post('content/get/random', {'batch_size': batch_size}) |