diff --git a/requirements-http.txt b/requirements-http.txt index 8be0a3e..67e8841 100644 --- a/requirements-http.txt +++ b/requirements-http.txt @@ -1,8 +1,9 @@ # requirements for swh.core.api aiohttp +aiohttp_utils >= 3.1.1 arrow decorator Flask msgpack > 0.5 python-dateutil requests diff --git a/swh/core/api/asynchronous.py b/swh/core/api/asynchronous.py index 93d5c70..761bce5 100644 --- a/swh/core/api/asynchronous.py +++ b/swh/core/api/asynchronous.py @@ -1,61 +1,96 @@ import json import logging import pickle import sys import traceback +from collections import OrderedDict +import multidict import aiohttp.web from deprecated import deprecated -import multidict -from .serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder +from .serializers import msgpack_dumps, msgpack_loads +from .serializers import SWHJSONDecoder, SWHJSONEncoder + +try: + from aiohttp_utils import negotiation, Response +except ImportError: + from aiohttp import Response + negotiation = None -def encode_data_server(data, **kwargs): +def encode_msgpack(data, **kwargs): return aiohttp.web.Response( body=msgpack_dumps(data), - headers=multidict.MultiDict({'Content-Type': 'application/x-msgpack'}), + headers=multidict.MultiDict( + {'Content-Type': 'application/x-msgpack'}), **kwargs ) +if negotiation is None: + encode_data_server = encode_msgpack +else: + encode_data_server = Response + + +def render_msgpack(request, data): + return msgpack_dumps(data) + + +def render_json(request, data): + return json.dumps(data, cls=SWHJSONEncoder) + + async def decode_request(request): - content_type = request.headers.get('Content-Type') + content_type = request.headers.get('Content-Type').split(';')[0].strip() data = await request.read() if not data: return {} if content_type == 'application/x-msgpack': r = msgpack_loads(data) elif content_type == 'application/json': r = json.loads(data.decode(), cls=SWHJSONDecoder) else: raise ValueError('Wrong content type `%s` for API request' % content_type) return r async def error_middleware(app, handler): async def middleware_handler(request): try: - return (await handler(request)) + return await handler(request) except Exception as e: if isinstance(e, aiohttp.web.HTTPException): raise logging.exception(e) exception = traceback.format_exception(*sys.exc_info()) res = {'exception': exception, 'exception_pickled': pickle.dumps(e)} return encode_data_server(res, status=500) return middleware_handler class RPCServerApp(aiohttp.web.Application): def __init__(self, *args, middlewares=(), **kwargs): middlewares = (error_middleware,) + middlewares + if negotiation: + # renderers are sorted in order of increasing desirability (!) + # see mimeparse.best_match() docstring. + renderers = OrderedDict([ + ('application/json', render_json), + ('application/x-msgpack', render_msgpack), + ]) + nego_middleware = negotiation.negotiation_middleware( + renderers=renderers, + force_rendering=True) + middlewares = (nego_middleware,) + middlewares + super().__init__(*args, middlewares=middlewares, **kwargs) @deprecated(version='0.0.64', reason='Use the RPCServerApp instead') class SWHRemoteAPI(RPCServerApp): pass diff --git a/swh/core/api/tests/test_async.py b/swh/core/api/tests/test_async.py index 883149e..96fec21 100644 --- a/swh/core/api/tests/test_async.py +++ b/swh/core/api/tests/test_async.py @@ -1,115 +1,187 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json import msgpack import pytest -from swh.core.api.asynchronous import RPCServerApp -from swh.core.api.asynchronous import encode_data_server, decode_request +from swh.core.api.asynchronous import RPCServerApp, Response +from swh.core.api.asynchronous import encode_msgpack, decode_request + from swh.core.api.serializers import msgpack_dumps, SWHJSONEncoder pytest_plugins = ['aiohttp.pytest_plugin', 'pytester'] async def root(request): - return encode_data_server('toor') + return Response('toor') STRUCT = {'txt': 'something stupid', # 'date': datetime.date(2019, 6, 9), # not supported 'datetime': datetime.datetime(2019, 6, 9, 10, 12), 'timedelta': datetime.timedelta(days=-2, hours=3), 'int': 42, 'float': 3.14, 'subdata': {'int': 42, 'datetime': datetime.datetime(2019, 6, 10, 11, 12), }, 'list': [42, datetime.datetime(2019, 9, 10, 11, 12), 'ok'], } async def struct(request): - return encode_data_server(STRUCT) + return Response(STRUCT) async def echo(request): data = await decode_request(request) - return encode_data_server(data) + return Response(data) + + +async def echo_no_nego(request): + # let the content negotiation handle the serialization for us... + data = await decode_request(request) + ret = encode_msgpack(data) + return ret + + +def check_mimetype(src, dst): + src = src.split(';')[0].strip() + dst = dst.split(';')[0].strip() + assert src == dst @pytest.fixture def app(): app = RPCServerApp() app.router.add_route('GET', '/', root) app.router.add_route('GET', '/struct', struct) app.router.add_route('POST', '/echo', echo) + app.router.add_route('POST', '/echo-no-nego', echo_no_nego) return app async def test_get_simple(app, aiohttp_client) -> None: assert app is not None cli = await aiohttp_client(app) resp = await cli.get('/') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') data = await resp.read() value = msgpack.unpackb(data, raw=False) assert value == 'toor' +async def test_get_simple_nego(app, aiohttp_client) -> None: + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/', headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == 'toor' + + async def test_get_struct(app, aiohttp_client) -> None: """Test returned structured from a simple GET data is OK""" cli = await aiohttp_client(app) resp = await cli.get('/struct') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT +async def test_get_struct_nego(app, aiohttp_client) -> None: + """Test returned structured from a simple GET data is OK""" + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/struct', + headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + async def test_post_struct_msgpack(app, aiohttp_client) -> None: """Test that msgpack encoded posted struct data is returned as is""" cli = await aiohttp_client(app) # simple struct resp = await cli.post( '/echo', headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps({'toto': 42})) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} # complex struct resp = await cli.post( '/echo', headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps(STRUCT)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT async def test_post_struct_json(app, aiohttp_client) -> None: """Test that json encoded posted struct data is returned as is""" cli = await aiohttp_client(app) resp = await cli.post( '/echo', headers={'Content-Type': 'application/json'}, data=json.dumps({'toto': 42}, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} resp = await cli.post( '/echo', headers={'Content-Type': 'application/json'}, data=json.dumps(STRUCT, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + # assert resp.headers['Content-Type'] == 'application/x-msgpack' assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as is + + using content negotiation (accept json or msgpack). + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_no_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as msgpack + + when using non-negotiation-compatible handlers. + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo-no-nego', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + assert (await decode_request(resp)) == STRUCT