Differential D10 Diff 54 swh/storage/objstorage/api/client.py

Changeset View

Standalone View

swh/storage/objstorage/api/client.py

This file was added.

				# Copyright (C) 2015 The Software Heritage developers
				# See the AUTHORS file at the top-level directory of this distribution
				# License: GNU General Public License version 3, or any later version
				# See top-level LICENSE file for more information


				import pickle

				import requests

				from requests.exceptions import ConnectionError
				from swh.core.serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder
				from swh.storage.exc import StorageAPIError

				olasdUnsubmitted Done Inline Actions Same comment about relative imports and the backslash. olasd: Same comment about relative imports and the backslash.

				def encode_data(data):
				try:
				return msgpack_dumps(data)
				except OverflowError as e:
				raise ValueError('Limits were reached. Please, check your input.\n' +
				str(e))


				def decode_response(response):
				content_type = response.headers['content-type']

				if content_type.startswith('application/x-msgpack'):
				r = msgpack_loads(response.content)
				elif content_type.startswith('application/json'):
				r = response.json(cls=SWHJSONDecoder)
				else:
				raise ValueError('Wrong content type `%s` for API response'
				% content_type)

				return r
				zackUnsubmitted Not Done Inline Actions This seems copy/pasted from storage/api/client.py. We should avoid that and rather refactor a common sub-module of swh-storage and have it used by both storage and objstorage clients. zack: This seems copy/pasted from storage/api/client.py. We should avoid that and rather refactor a…


				class RemoteArchive():
				""" proxy to a remote archive storage. """
				def __init__(self, base_url):
				self.base_url = base_url
				self.session = requests.Session()

				def url(self, endpoint):
				return '%s%s' % (self.base_url, endpoint)

				def post(self, endpoint, data):
				try:
				response = self.session.post(
				self.url(endpoint),
				data=encode_data(data),
				headers={'content-type': 'application/x-msgpack'},
				)
				except ConnectionError as e:
				print(str(e))
				raise StorageAPIError(e)

				# XXX: this breaks language-independence and should be
				# replaced by proper unserialization
				if response.status_code == 400:
				raise pickle.loads(decode_response(response))

				return decode_response(response)

				def content_add(self, bytes, obj_id=None):
				""" Add a new object to the object storage.

				Args:
				bytes: content of the object to be added to the storage.
				obj_id: checksums of `bytes` as computed by ID_HASH_ALGO. When
				given, obj_id will be trusted to match bytes. If missing,
				obj_id will be computed on the fly.

				"""
				return self.post('content/add', {'bytes': bytes, 'obj_id': obj_id})

				def content_get(self, obj_id):
				""" Retrieve the content of a given object.

				Args:
				obj_id: The id of the object.

				Returns:
				The content of the requested objects as bytes.

				Raises:
				ObjNotFoundError: if the requested object is missing
				"""
				return self.post('content/get', {'obj_id': obj_id})

				def content_check(self, obj_id):
				""" Integrity check for a given object

				verify that the file object is in place, and that the gzipped content
				matches the object id

				Args:
				obj_id: The id of the object.

				Raises:
				ObjNotFoundError: if the requested object is missing
				Error: if the requested object is corrupt
				"""
				self.post('content/check', {'obj_id': obj_id})