diff --git a/requirements-http.txt b/requirements-http.txt --- a/requirements-http.txt +++ b/requirements-http.txt @@ -1,5 +1,6 @@ # requirements for swh.core.api aiohttp +aiohttp_utils >= 3.1.1 arrow decorator Flask diff --git a/swh/core/api/asynchronous.py b/swh/core/api/asynchronous.py --- a/swh/core/api/asynchronous.py +++ b/swh/core/api/asynchronous.py @@ -3,24 +3,47 @@ import pickle import sys import traceback +from collections import OrderedDict +import multidict import aiohttp.web from deprecated import deprecated -import multidict -from .serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder +from .serializers import msgpack_dumps, msgpack_loads +from .serializers import SWHJSONDecoder, SWHJSONEncoder + +try: + from aiohttp_utils import negotiation, Response +except ImportError: + from aiohttp import Response + negotiation = None -def encode_data_server(data, **kwargs): +def encode_msgpack(data, **kwargs): return aiohttp.web.Response( body=msgpack_dumps(data), - headers=multidict.MultiDict({'Content-Type': 'application/x-msgpack'}), + headers=multidict.MultiDict( + {'Content-Type': 'application/x-msgpack'}), **kwargs ) +if negotiation is None: + encode_data_server = encode_msgpack +else: + encode_data_server = Response + + +def render_msgpack(request, data): + return msgpack_dumps(data) + + +def render_json(request, data): + return json.dumps(data, cls=SWHJSONEncoder) + + async def decode_request(request): - content_type = request.headers.get('Content-Type') + content_type = request.headers.get('Content-Type').split(';')[0].strip() data = await request.read() if not data: return {} @@ -37,7 +60,7 @@ async def error_middleware(app, handler): async def middleware_handler(request): try: - return (await handler(request)) + return await handler(request) except Exception as e: if isinstance(e, aiohttp.web.HTTPException): raise @@ -52,6 +75,18 @@ class RPCServerApp(aiohttp.web.Application): def __init__(self, *args, middlewares=(), **kwargs): middlewares = (error_middleware,) + middlewares + if negotiation: + # renderers are sorted in order of increasing desirability (!) + # see mimeparse.best_match() docstring. + renderers = OrderedDict([ + ('application/json', render_json), + ('application/x-msgpack', render_msgpack), + ]) + nego_middleware = negotiation.negotiation_middleware( + renderers=renderers, + force_rendering=True) + middlewares = (nego_middleware,) + middlewares + super().__init__(*args, middlewares=middlewares, **kwargs) diff --git a/swh/core/api/tests/test_async.py b/swh/core/api/tests/test_async.py --- a/swh/core/api/tests/test_async.py +++ b/swh/core/api/tests/test_async.py @@ -10,8 +10,9 @@ import pytest -from swh.core.api.asynchronous import RPCServerApp -from swh.core.api.asynchronous import encode_data_server, decode_request +from swh.core.api.asynchronous import RPCServerApp, Response +from swh.core.api.asynchronous import encode_msgpack, decode_request + from swh.core.api.serializers import msgpack_dumps, SWHJSONEncoder @@ -19,7 +20,7 @@ async def root(request): - return encode_data_server('toor') + return Response('toor') STRUCT = {'txt': 'something stupid', # 'date': datetime.date(2019, 6, 9), # not supported @@ -35,12 +36,25 @@ async def struct(request): - return encode_data_server(STRUCT) + return Response(STRUCT) async def echo(request): data = await decode_request(request) - return encode_data_server(data) + return Response(data) + + +async def echo_no_nego(request): + # let the content negotiation handle the serialization for us... + data = await decode_request(request) + ret = encode_msgpack(data) + return ret + + +def check_mimetype(src, dst): + src = src.split(';')[0].strip() + dst = dst.split(';')[0].strip() + assert src == dst @pytest.fixture @@ -49,6 +63,7 @@ app.router.add_route('GET', '/', root) app.router.add_route('GET', '/struct', struct) app.router.add_route('POST', '/echo', echo) + app.router.add_route('POST', '/echo-no-nego', echo_no_nego) return app @@ -58,21 +73,41 @@ cli = await aiohttp_client(app) resp = await cli.get('/') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') data = await resp.read() value = msgpack.unpackb(data, raw=False) assert value == 'toor' +async def test_get_simple_nego(app, aiohttp_client) -> None: + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/', headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == 'toor' + + async def test_get_struct(app, aiohttp_client) -> None: """Test returned structured from a simple GET data is OK""" cli = await aiohttp_client(app) resp = await cli.get('/struct') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT +async def test_get_struct_nego(app, aiohttp_client) -> None: + """Test returned structured from a simple GET data is OK""" + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/struct', + headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + async def test_post_struct_msgpack(app, aiohttp_client) -> None: """Test that msgpack encoded posted struct data is returned as is""" cli = await aiohttp_client(app) @@ -82,7 +117,7 @@ headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps({'toto': 42})) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} # complex struct resp = await cli.post( @@ -90,7 +125,7 @@ headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps(STRUCT)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT @@ -103,7 +138,7 @@ headers={'Content-Type': 'application/json'}, data=json.dumps({'toto': 42}, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} resp = await cli.post( @@ -111,5 +146,42 @@ headers={'Content-Type': 'application/json'}, data=json.dumps(STRUCT, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + # assert resp.headers['Content-Type'] == 'application/x-msgpack' assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as is + + using content negotiation (accept json or msgpack). + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_no_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as msgpack + + when using non-negotiation-compatible handlers. + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo-no-nego', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + assert (await decode_request(resp)) == STRUCT