Changeset View
Changeset View
Standalone View
Standalone View
swh/core/api.py
# Copyright (C) 2015-2017 The Software Heritage developers | # Copyright (C) 2015-2017 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import collections | import collections | ||||
import functools | import functools | ||||
import inspect | import inspect | ||||
import json | import json | ||||
import logging | import logging | ||||
import pickle | import pickle | ||||
import requests | import requests | ||||
import datetime | |||||
from flask import Flask, Request, Response | from flask import Flask, Request, Response | ||||
from .serializers import (decode_response, | from .serializers import (decode_response, | ||||
encode_data_client as encode_data, | encode_data_client as encode_data, | ||||
msgpack_dumps, msgpack_loads, SWHJSONDecoder) | msgpack_dumps, msgpack_loads, SWHJSONDecoder) | ||||
from negotiate.flask import Formatter | |||||
logger = logging.getLogger(__name__) | |||||
class SWHJSONEncoder(json.JSONEncoder): | |||||
def default(self, obj): | |||||
if isinstance(obj, (datetime.datetime, datetime.date)): | |||||
return obj.isoformat() | |||||
if isinstance(obj, datetime.timedelta): | |||||
return str(obj) | |||||
# Let the base class default method raise the TypeError | |||||
ardumont: That won't be enough for all of our internal apis (but i have the feeling you know that and… | |||||
Done Inline ActionsYes,so far this is enough for the scheduler (diff not yet submitted), but indeed it will need more tweaking... douardda: Yes,so far this is enough for the scheduler (diff not yet submitted), but indeed it will need… | |||||
return super().default(obj) | |||||
class JSONFormatter(Formatter): | |||||
format = 'json' | |||||
mimetypes = ['application/json'] | |||||
def render(self, obj): | |||||
return json.dumps(obj, cls=SWHJSONEncoder) | |||||
class MsgpackFormatter(Formatter): | |||||
format = 'msgpack' | |||||
mimetypes = ['application/x-msgpack'] | |||||
def render(self, obj): | |||||
return msgpack_dumps(obj) | |||||
class RemoteException(Exception): | class RemoteException(Exception): | ||||
pass | pass | ||||
def remote_api_endpoint(path): | def remote_api_endpoint(path): | ||||
def dec(f): | def dec(f): | ||||
f._endpoint_path = path | f._endpoint_path = path | ||||
▲ Show 20 Lines • Show All 92 Lines • ▼ Show 20 Lines | def raw_get(self, endpoint, params=None, **opts): | ||||
) | ) | ||||
except requests.exceptions.ConnectionError as e: | except requests.exceptions.ConnectionError as e: | ||||
raise self.api_exception(e) | raise self.api_exception(e) | ||||
def post(self, endpoint, data, params=None): | def post(self, endpoint, data, params=None): | ||||
data = encode_data(data) | data = encode_data(data) | ||||
response = self.raw_post( | response = self.raw_post( | ||||
endpoint, data, params=params, | endpoint, data, params=params, | ||||
headers={'content-type': 'application/x-msgpack'}) | headers={'content-type': 'application/x-msgpack', | ||||
Not Done Inline Actionswhy did you add that part? ardumont: why did you add that part?
That confuses me, i'm checking ;) | |||||
Done Inline ActionsIn the context of an API client (remote service), we want to make sure we ask the API server for msgpack encoded response. The content-type only declare we send msgpack encoded payload, not what we ask for in return. douardda: In the context of an API client (remote service), we want to make sure we ask the API server… | |||||
Not Done Inline Actionsyes, remote! I misread. I keep forgetting that this code is the client part ;) In the end, this was missing instruction all along (independent from the negotiate library introduction here). Thanks. ardumont: yes, remote! I misread. I keep forgetting that this code is the client part ;)
In the end… | |||||
'accept': 'application/x-msgpack'}) | |||||
return self._decode_response(response) | return self._decode_response(response) | ||||
def get(self, endpoint, params=None): | def get(self, endpoint, params=None): | ||||
response = self.raw_get(endpoint, params=params) | response = self.raw_get( | ||||
endpoint, params=params, | |||||
headers={'accept': 'application/x-msgpack'}) | |||||
return self._decode_response(response) | return self._decode_response(response) | ||||
def post_stream(self, endpoint, data, params=None): | def post_stream(self, endpoint, data, params=None): | ||||
if not isinstance(data, collections.Iterable): | if not isinstance(data, collections.Iterable): | ||||
raise ValueError("`data` must be Iterable") | raise ValueError("`data` must be Iterable") | ||||
response = self.raw_post(endpoint, data, params=params) | response = self.raw_post( | ||||
endpoint, data, params=params, | |||||
headers={'accept': 'application/x-msgpack'}) | |||||
return self._decode_response(response) | return self._decode_response(response) | ||||
def get_stream(self, endpoint, params=None, chunk_size=4096): | def get_stream(self, endpoint, params=None, chunk_size=4096): | ||||
response = self.raw_get(endpoint, params=params, stream=True) | response = self.raw_get(endpoint, params=params, stream=True, | ||||
headers={'accept': 'application/x-msgpack'}) | |||||
return response.iter_content(chunk_size) | return response.iter_content(chunk_size) | ||||
def _decode_response(self, response): | def _decode_response(self, response): | ||||
if response.status_code == 404: | if response.status_code == 404: | ||||
return None | return None | ||||
if response.status_code == 500: | if response.status_code == 500: | ||||
data = decode_response(response) | data = decode_response(response) | ||||
if 'exception_pickled' in data: | if 'exception_pickled' in data: | ||||
Show All 16 Lines | |||||
class BytesRequest(Request): | class BytesRequest(Request): | ||||
"""Request with proper escaping of arbitrary byte sequences.""" | """Request with proper escaping of arbitrary byte sequences.""" | ||||
encoding = 'utf-8' | encoding = 'utf-8' | ||||
encoding_errors = 'surrogateescape' | encoding_errors = 'surrogateescape' | ||||
def encode_data_server(data): | ENCODERS = { | ||||
'application/x-msgpack': msgpack_dumps, | |||||
'application/json': json.dumps, | |||||
} | |||||
def encode_data_server(data, content_type='application/x-msgpack'): | |||||
encoded_data = ENCODERS[content_type](data) | |||||
return Response( | return Response( | ||||
msgpack_dumps(data), | encoded_data, | ||||
mimetype='application/x-msgpack', | mimetype=content_type, | ||||
) | ) | ||||
def decode_request(request): | def decode_request(request): | ||||
content_type = request.mimetype | content_type = request.mimetype | ||||
data = request.get_data() | data = request.get_data() | ||||
if not data: | |||||
return {} | |||||
if content_type == 'application/x-msgpack': | if content_type == 'application/x-msgpack': | ||||
r = msgpack_loads(data) | r = msgpack_loads(data) | ||||
elif content_type == 'application/json': | elif content_type == 'application/json': | ||||
r = json.loads(data, cls=SWHJSONDecoder) | r = json.loads(data, cls=SWHJSONDecoder) | ||||
else: | else: | ||||
raise ValueError('Wrong content type `%s` for API request' | raise ValueError('Wrong content type `%s` for API request' | ||||
% content_type) | % content_type) | ||||
▲ Show 20 Lines • Show All 46 Lines • Show Last 20 Lines |
That won't be enough for all of our internal apis (but i have the feeling you know that and want to go incrementally ;)
I think of the storage for example but there might be others as well (indexer?).