Changeset View
Changeset View
Standalone View
Standalone View
swh/core/api/__init__.py
| # Copyright (C) 2015-2017 The Software Heritage developers | # Copyright (C) 2015-2017 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from collections import abc | from collections import abc | ||||
| import datetime | |||||
| import functools | import functools | ||||
| import inspect | import inspect | ||||
| import json | import json | ||||
| import logging | import logging | ||||
| import pickle | import pickle | ||||
| import requests | import requests | ||||
| import datetime | import traceback | ||||
| from typing import Any, ClassVar, Optional, Type | from typing import Any, ClassVar, List, Optional, Type | ||||
| from flask import Flask, Request, Response, request, abort | from flask import Flask, Request, Response, request, abort | ||||
| from werkzeug.exceptions import HTTPException | |||||
| from .serializers import (decode_response, | from .serializers import (decode_response, | ||||
| encode_data_client as encode_data, | encode_data_client as encode_data, | ||||
| msgpack_dumps, msgpack_loads, SWHJSONDecoder) | msgpack_dumps, msgpack_loads, SWHJSONDecoder) | ||||
| from .negotiation import (Formatter as FormatterBase, | from .negotiation import (Formatter as FormatterBase, | ||||
| Negotiator as NegotiatorBase, | Negotiator as NegotiatorBase, | ||||
| negotiate as _negotiate) | negotiate as _negotiate) | ||||
| ▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | class RemoteException(Exception): | ||||
| def __init__(self, payload: Optional[Any] = None, | def __init__(self, payload: Optional[Any] = None, | ||||
| response: Optional[requests.Response] = None): | response: Optional[requests.Response] = None): | ||||
| if payload is not None: | if payload is not None: | ||||
| super().__init__(payload) | super().__init__(payload) | ||||
| else: | else: | ||||
| super().__init__() | super().__init__() | ||||
| self.response = response | self.response = response | ||||
| def __str__(self): | |||||
ardumont: is self.args systematically a list? | |||||
Done Inline ActionsIt's a tuple, but yes vlorentz: It's a tuple, but yes | |||||
| if self.args and isinstance(self.args[0], dict) \ | |||||
| and 'type' in self.args[0] and 'args' in self.args[0]: | |||||
| return ( | |||||
| f'<RemoteException {self.response.status_code} ' | |||||
| f'{self.args[0]["type"]}: {self.args[0]["args"]}>') | |||||
| else: | |||||
| return super().__str__() | |||||
| def remote_api_endpoint(path): | def remote_api_endpoint(path): | ||||
| def dec(f): | def dec(f): | ||||
| f._endpoint_path = path | f._endpoint_path = path | ||||
| return f | return f | ||||
| return dec | return dec | ||||
| ▲ Show 20 Lines • Show All 60 Lines • ▼ Show 20 Lines | class RPCClient(metaclass=MetaRPCClient): | ||||
| This backend class will never be instantiated, it only serves as | This backend class will never be instantiated, it only serves as | ||||
| a template.""" | a template.""" | ||||
| api_exception = APIError # type: ClassVar[Type[Exception]] | api_exception = APIError # type: ClassVar[Type[Exception]] | ||||
| """The exception class to raise in case of communication error with | """The exception class to raise in case of communication error with | ||||
| the server.""" | the server.""" | ||||
| reraise_exceptions: ClassVar[List[Type[Exception]]] = [] | |||||
| """On server errors, if any of the exception classes in this list | |||||
| has the same name as the error name, then the exception will | |||||
| be instantiated and raised instead of a generic RemoteException.""" | |||||
| def __init__(self, url, api_exception=None, | def __init__(self, url, api_exception=None, | ||||
| timeout=None, chunk_size=4096, **kwargs): | timeout=None, chunk_size=4096, | ||||
| reraise_exceptions=None, **kwargs): | |||||
| if api_exception: | if api_exception: | ||||
| self.api_exception = api_exception | self.api_exception = api_exception | ||||
| if reraise_exceptions: | |||||
| self.reraise_exceptions = reraise_exceptions | |||||
| base_url = url if url.endswith('/') else url + '/' | base_url = url if url.endswith('/') else url + '/' | ||||
| self.url = base_url | self.url = base_url | ||||
| self.session = requests.Session() | self.session = requests.Session() | ||||
| adapter = requests.adapters.HTTPAdapter( | adapter = requests.adapters.HTTPAdapter( | ||||
| max_retries=kwargs.get('max_retries', 3), | max_retries=kwargs.get('max_retries', 3), | ||||
| pool_connections=kwargs.get('pool_connections', 20), | pool_connections=kwargs.get('pool_connections', 20), | ||||
| pool_maxsize=kwargs.get('pool_maxsize', 100)) | pool_maxsize=kwargs.get('pool_maxsize', 100)) | ||||
| self.session.mount(self.url, adapter) | self.session.mount(self.url, adapter) | ||||
| ▲ Show 20 Lines • Show All 55 Lines • ▼ Show 20 Lines | class RPCClient(metaclass=MetaRPCClient): | ||||
| def get_stream(self, endpoint, **opts): | def get_stream(self, endpoint, **opts): | ||||
| return self.get(endpoint, stream=True, **opts) | return self.get(endpoint, stream=True, **opts) | ||||
| def raise_for_status(self, response) -> None: | def raise_for_status(self, response) -> None: | ||||
| """check response HTTP status code and raise an exception if it denotes an | """check response HTTP status code and raise an exception if it denotes an | ||||
| error; do nothing otherwise | error; do nothing otherwise | ||||
| """ | """ | ||||
| # XXX: unpickling below breaks language-independence and should be | |||||
| # replaced by proper language-independent [de]serialization | |||||
| status_code = response.status_code | status_code = response.status_code | ||||
| status_class = response.status_code // 100 | status_class = response.status_code // 100 | ||||
| if status_code == 404: | if status_code == 404: | ||||
| raise RemoteException(payload='404 not found', response=response) | raise RemoteException(payload='404 not found', response=response) | ||||
| exception = None | exception = None | ||||
| # TODO: only old servers send pickled error; stop trying to unpickle | |||||
| # after they are all upgraded | |||||
| try: | try: | ||||
| if status_class == 4: | if status_class == 4: | ||||
| data = decode_response(response) | data = decode_response(response) | ||||
| if isinstance(data, dict): | |||||
| for exc_type in self.reraise_exceptions: | |||||
| if exc_type.__name__ == data['exception']['type']: | |||||
| exception = exc_type(*data['exception']['args']) | |||||
| break | |||||
| else: | |||||
| exception = RemoteException(payload=data['exception'], | |||||
| response=response) | |||||
| else: | |||||
| exception = pickle.loads(data) | exception = pickle.loads(data) | ||||
| elif status_class == 5: | elif status_class == 5: | ||||
| data = decode_response(response) | data = decode_response(response) | ||||
| if 'exception_pickled' in data: | if 'exception_pickled' in data: | ||||
| exception = pickle.loads(data['exception_pickled']) | exception = pickle.loads(data['exception_pickled']) | ||||
| else: | else: | ||||
| exception = RemoteException(payload=data['exception'], | exception = RemoteException(payload=data['exception'], | ||||
| response=response) | response=response) | ||||
| except (TypeError, pickle.UnpicklingError): | except (TypeError, pickle.UnpicklingError): | ||||
| raise RemoteException(payload=data, response=response) | raise RemoteException(payload=data, response=response) | ||||
| if exception: | if exception: | ||||
| raise exception from None | raise exception from None | ||||
| if status_class != 2: | if status_class != 2: | ||||
| raise RemoteException( | raise RemoteException( | ||||
| payload=f'API HTTP error: {status_code} {response.content}', | payload=f'API HTTP error: {status_code} {response.content}', | ||||
| response=response) | response=response) | ||||
| def _decode_response(self, response): | def _decode_response(self, response): | ||||
| if response.status_code == 404: | |||||
| return None | |||||
| else: | |||||
| self.raise_for_status(response) | self.raise_for_status(response) | ||||
| return decode_response(response) | return decode_response(response) | ||||
Not Done Inline Actionsunsure about removing that statement. ardumont: unsure about removing that statement.
i guess that's the price for not having tests on that… | |||||
Done Inline ActionsIt's a bad idea to return None in case an endpoint doesn't exist, anyway. vlorentz: It's a bad idea to return None in case an endpoint doesn't exist, anyway. | |||||
| def __repr__(self): | def __repr__(self): | ||||
| return '<{} url={}>'.format(self.__class__.__name__, self.url) | return '<{} url={}>'.format(self.__class__.__name__, self.url) | ||||
| class BytesRequest(Request): | class BytesRequest(Request): | ||||
| """Request with proper escaping of arbitrary byte sequences.""" | """Request with proper escaping of arbitrary byte sequences.""" | ||||
| encoding = 'utf-8' | encoding = 'utf-8' | ||||
| Show All 28 Lines | elif content_type == 'application/json': | ||||
| r = json.loads(data.decode('utf-8'), cls=SWHJSONDecoder) | r = json.loads(data.decode('utf-8'), cls=SWHJSONDecoder) | ||||
| else: | else: | ||||
| raise ValueError('Wrong content type `%s` for API request' | raise ValueError('Wrong content type `%s` for API request' | ||||
| % content_type) | % content_type) | ||||
| return r | return r | ||||
| def error_handler(exception, encoder): | def error_handler(exception, encoder, status_code=500): | ||||
| # XXX: this breaks language-independence and should be | |||||
| # replaced by proper serialization of errors | |||||
| logging.exception(exception) | logging.exception(exception) | ||||
| response = encoder(pickle.dumps(exception)) | tb = traceback.format_exception(None, exception, exception.__traceback__) | ||||
| response.status_code = 400 | error = { | ||||
| 'exception': { | |||||
| 'type': type(exception).__name__, | |||||
| 'args': exception.args, | |||||
| 'message': str(exception), | |||||
| 'traceback': tb, | |||||
| } | |||||
| } | |||||
| response = encoder(error) | |||||
| if isinstance(exception, HTTPException): | |||||
| response.status_code = exception.code | |||||
| else: | |||||
| # TODO: differentiate between server errors and client errors | |||||
| response.status_code = status_code | |||||
| return response | return response | ||||
| class RPCServerApp(Flask): | class RPCServerApp(Flask): | ||||
| """For each endpoint of the given `backend_class`, tells app.route to call | """For each endpoint of the given `backend_class`, tells app.route to call | ||||
| a function that decodes the request and sends it to the backend object | a function that decodes the request and sends it to the backend object | ||||
| provided by the factory. | provided by the factory. | ||||
| Show All 34 Lines | |||||
is self.args systematically a list?