diff --git a/PKG-INFO b/PKG-INFO index 04eec84..f943a46 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,88 +1,88 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.64 +Version: 0.0.65 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest -Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing Provides-Extra: http Provides-Extra: db +Provides-Extra: testing diff --git a/requirements-http.txt b/requirements-http.txt index 8be0a3e..67e8841 100644 --- a/requirements-http.txt +++ b/requirements-http.txt @@ -1,8 +1,9 @@ # requirements for swh.core.api aiohttp +aiohttp_utils >= 3.1.1 arrow decorator Flask msgpack > 0.5 python-dateutil requests diff --git a/swh.core.egg-info/PKG-INFO b/swh.core.egg-info/PKG-INFO index 04eec84..f943a46 100644 --- a/swh.core.egg-info/PKG-INFO +++ b/swh.core.egg-info/PKG-INFO @@ -1,88 +1,88 @@ Metadata-Version: 2.1 Name: swh.core -Version: 0.0.64 +Version: 0.0.65 Summary: Software Heritage core utilities Home-page: https://forge.softwareheritage.org/diffusion/DCORE/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest -Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Project-URL: Funding, https://www.softwareheritage.org/donate +Project-URL: Source, https://forge.softwareheritage.org/source/swh-core Description: swh-core ======== core library for swh's modules: - config parser - hash computations - serialization - logging mechanism - database connection - http-based RPC client/server Development ----------- We strongly recommend you to use a [virtualenv][1] if you want to run tests or hack the code. To set up your development environment: ``` (swh) user@host:~/swh-environment/swh-core$ pip install -e .[testing] ``` This will install every Python package needed to run this package's tests. Unit tests can be executed using [pytest][2] or [tox][3]. ``` (swh) user@host:~/swh-environment/swh-core$ pytest ============================== test session starts ============================== platform linux -- Python 3.7.3, pytest-3.10.1, py-1.8.0, pluggy-0.12.0 hypothesis profile 'default' -> database=DirectoryBasedExampleDatabase('/home/ddouard/src/swh-environment/swh-core/.hypothesis/examples') rootdir: /home/ddouard/src/swh-environment/swh-core, inifile: pytest.ini plugins: requests-mock-1.6.0, hypothesis-4.26.4, celery-4.3.0, postgresql-1.4.1 collected 89 items swh/core/api/tests/test_api.py .. [ 2%] swh/core/api/tests/test_async.py .... [ 6%] swh/core/api/tests/test_serializers.py ..... [ 12%] swh/core/db/tests/test_db.py .... [ 16%] swh/core/tests/test_cli.py ...... [ 23%] swh/core/tests/test_config.py .............. [ 39%] swh/core/tests/test_statsd.py ........................................... [ 87%] .... [ 92%] swh/core/tests/test_utils.py ....... [100%] ===================== 89 passed, 9 warnings in 6.94 seconds ===================== ``` Note: this git repository uses [pre-commit][4] hooks to ensure better and more consistent code. It should already be installed in your virtualenv (if not, just type `pip install pre-commit`). Make sure to activate it in your local copy of the git repository: ``` (swh) user@host:~/swh-environment/swh-core$ pre-commit install pre-commit installed at .git/hooks/pre-commit ``` Please read the [developer setup manual][5] for more information on how to hack on Software Heritage. [1]: https://virtualenv.pypa.io [2]: https://docs.pytest.org [3]: https://tox.readthedocs.io [4]: https://pre-commit.com [5]: https://docs.softwareheritage.org/devel/developer-setup.html Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Description-Content-Type: text/markdown -Provides-Extra: testing Provides-Extra: http Provides-Extra: db +Provides-Extra: testing diff --git a/swh.core.egg-info/requires.txt b/swh.core.egg-info/requires.txt index 854035c..dbc39ec 100644 --- a/swh.core.egg-info/requires.txt +++ b/swh.core.egg-info/requires.txt @@ -1,31 +1,33 @@ Deprecated PyYAML systemd-python [db] psycopg2 [http] aiohttp +aiohttp_utils>=3.1.1 arrow decorator Flask msgpack>0.5 python-dateutil requests [testing] Click pytest<4 pytest-postgresql requests-mock hypothesis>=3.11.0 pre-commit psycopg2 aiohttp +aiohttp_utils>=3.1.1 arrow decorator Flask msgpack>0.5 python-dateutil requests diff --git a/swh/core/api/asynchronous.py b/swh/core/api/asynchronous.py index 93d5c70..761bce5 100644 --- a/swh/core/api/asynchronous.py +++ b/swh/core/api/asynchronous.py @@ -1,61 +1,96 @@ import json import logging import pickle import sys import traceback +from collections import OrderedDict +import multidict import aiohttp.web from deprecated import deprecated -import multidict -from .serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder +from .serializers import msgpack_dumps, msgpack_loads +from .serializers import SWHJSONDecoder, SWHJSONEncoder + +try: + from aiohttp_utils import negotiation, Response +except ImportError: + from aiohttp import Response + negotiation = None -def encode_data_server(data, **kwargs): +def encode_msgpack(data, **kwargs): return aiohttp.web.Response( body=msgpack_dumps(data), - headers=multidict.MultiDict({'Content-Type': 'application/x-msgpack'}), + headers=multidict.MultiDict( + {'Content-Type': 'application/x-msgpack'}), **kwargs ) +if negotiation is None: + encode_data_server = encode_msgpack +else: + encode_data_server = Response + + +def render_msgpack(request, data): + return msgpack_dumps(data) + + +def render_json(request, data): + return json.dumps(data, cls=SWHJSONEncoder) + + async def decode_request(request): - content_type = request.headers.get('Content-Type') + content_type = request.headers.get('Content-Type').split(';')[0].strip() data = await request.read() if not data: return {} if content_type == 'application/x-msgpack': r = msgpack_loads(data) elif content_type == 'application/json': r = json.loads(data.decode(), cls=SWHJSONDecoder) else: raise ValueError('Wrong content type `%s` for API request' % content_type) return r async def error_middleware(app, handler): async def middleware_handler(request): try: - return (await handler(request)) + return await handler(request) except Exception as e: if isinstance(e, aiohttp.web.HTTPException): raise logging.exception(e) exception = traceback.format_exception(*sys.exc_info()) res = {'exception': exception, 'exception_pickled': pickle.dumps(e)} return encode_data_server(res, status=500) return middleware_handler class RPCServerApp(aiohttp.web.Application): def __init__(self, *args, middlewares=(), **kwargs): middlewares = (error_middleware,) + middlewares + if negotiation: + # renderers are sorted in order of increasing desirability (!) + # see mimeparse.best_match() docstring. + renderers = OrderedDict([ + ('application/json', render_json), + ('application/x-msgpack', render_msgpack), + ]) + nego_middleware = negotiation.negotiation_middleware( + renderers=renderers, + force_rendering=True) + middlewares = (nego_middleware,) + middlewares + super().__init__(*args, middlewares=middlewares, **kwargs) @deprecated(version='0.0.64', reason='Use the RPCServerApp instead') class SWHRemoteAPI(RPCServerApp): pass diff --git a/swh/core/api/tests/test_async.py b/swh/core/api/tests/test_async.py index 883149e..96fec21 100644 --- a/swh/core/api/tests/test_async.py +++ b/swh/core/api/tests/test_async.py @@ -1,115 +1,187 @@ # Copyright (C) 2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import datetime import json import msgpack import pytest -from swh.core.api.asynchronous import RPCServerApp -from swh.core.api.asynchronous import encode_data_server, decode_request +from swh.core.api.asynchronous import RPCServerApp, Response +from swh.core.api.asynchronous import encode_msgpack, decode_request + from swh.core.api.serializers import msgpack_dumps, SWHJSONEncoder pytest_plugins = ['aiohttp.pytest_plugin', 'pytester'] async def root(request): - return encode_data_server('toor') + return Response('toor') STRUCT = {'txt': 'something stupid', # 'date': datetime.date(2019, 6, 9), # not supported 'datetime': datetime.datetime(2019, 6, 9, 10, 12), 'timedelta': datetime.timedelta(days=-2, hours=3), 'int': 42, 'float': 3.14, 'subdata': {'int': 42, 'datetime': datetime.datetime(2019, 6, 10, 11, 12), }, 'list': [42, datetime.datetime(2019, 9, 10, 11, 12), 'ok'], } async def struct(request): - return encode_data_server(STRUCT) + return Response(STRUCT) async def echo(request): data = await decode_request(request) - return encode_data_server(data) + return Response(data) + + +async def echo_no_nego(request): + # let the content negotiation handle the serialization for us... + data = await decode_request(request) + ret = encode_msgpack(data) + return ret + + +def check_mimetype(src, dst): + src = src.split(';')[0].strip() + dst = dst.split(';')[0].strip() + assert src == dst @pytest.fixture def app(): app = RPCServerApp() app.router.add_route('GET', '/', root) app.router.add_route('GET', '/struct', struct) app.router.add_route('POST', '/echo', echo) + app.router.add_route('POST', '/echo-no-nego', echo_no_nego) return app async def test_get_simple(app, aiohttp_client) -> None: assert app is not None cli = await aiohttp_client(app) resp = await cli.get('/') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') data = await resp.read() value = msgpack.unpackb(data, raw=False) assert value == 'toor' +async def test_get_simple_nego(app, aiohttp_client) -> None: + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/', headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == 'toor' + + async def test_get_struct(app, aiohttp_client) -> None: """Test returned structured from a simple GET data is OK""" cli = await aiohttp_client(app) resp = await cli.get('/struct') assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT +async def test_get_struct_nego(app, aiohttp_client) -> None: + """Test returned structured from a simple GET data is OK""" + cli = await aiohttp_client(app) + for ctype in ('x-msgpack', 'json'): + resp = await cli.get('/struct', + headers={'Accept': 'application/%s' % ctype}) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + async def test_post_struct_msgpack(app, aiohttp_client) -> None: """Test that msgpack encoded posted struct data is returned as is""" cli = await aiohttp_client(app) # simple struct resp = await cli.post( '/echo', headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps({'toto': 42})) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} # complex struct resp = await cli.post( '/echo', headers={'Content-Type': 'application/x-msgpack'}, data=msgpack_dumps(STRUCT)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == STRUCT async def test_post_struct_json(app, aiohttp_client) -> None: """Test that json encoded posted struct data is returned as is""" cli = await aiohttp_client(app) resp = await cli.post( '/echo', headers={'Content-Type': 'application/json'}, data=json.dumps({'toto': 42}, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') assert (await decode_request(resp)) == {'toto': 42} resp = await cli.post( '/echo', headers={'Content-Type': 'application/json'}, data=json.dumps(STRUCT, cls=SWHJSONEncoder)) assert resp.status == 200 - assert resp.headers['Content-Type'] == 'application/x-msgpack' + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + # assert resp.headers['Content-Type'] == 'application/x-msgpack' assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as is + + using content negotiation (accept json or msgpack). + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/%s' % ctype) + assert (await decode_request(resp)) == STRUCT + + +async def test_post_struct_no_nego(app, aiohttp_client) -> None: + """Test that json encoded posted struct data is returned as msgpack + + when using non-negotiation-compatible handlers. + """ + cli = await aiohttp_client(app) + + for ctype in ('x-msgpack', 'json'): + resp = await cli.post( + '/echo-no-nego', + headers={'Content-Type': 'application/json', + 'Accept': 'application/%s' % ctype}, + data=json.dumps(STRUCT, cls=SWHJSONEncoder)) + assert resp.status == 200 + check_mimetype(resp.headers['Content-Type'], 'application/x-msgpack') + assert (await decode_request(resp)) == STRUCT diff --git a/version.txt b/version.txt index 34057bb..cb5ddff 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.64-0-g13c18d7 \ No newline at end of file +v0.0.65-0-g4a20ead \ No newline at end of file