diff --git a/swh/storage/api/client.py b/swh/storage/api/client.py index 125f132c..85797210 100644 --- a/swh/storage/api/client.py +++ b/swh/storage/api/client.py @@ -1,41 +1,45 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from swh.core.api import RPCClient, RemoteException from .. import HashCollision from ..exc import StorageAPIError, StorageArgumentException from ..interface import StorageInterface +from .serializers import ENCODERS, DECODERS + class RemoteStorage(RPCClient): """Proxy to a remote storage API""" api_exception = StorageAPIError backend_class = StorageInterface reraise_exceptions = [ StorageArgumentException, ] + extra_type_decoders = DECODERS + extra_type_encoders = ENCODERS def raise_for_status(self, response) -> None: try: super().raise_for_status(response) except RemoteException as e: if e.response is not None and e.response.status_code == 500 \ and e.args and e.args[0].get('type') == 'HashCollision': # XXX: workaround until we fix these HashCollisions happening # when they shouldn't raise HashCollision( *e.args[0]['args']) else: raise def reset(self): return self.post('reset', {}) def stat_counters(self): return self.get('stat/counters') def refresh_stat_counters(self): return self.get('stat/refresh') diff --git a/swh/storage/api/serializers.py b/swh/storage/api/serializers.py new file mode 100644 index 00000000..b5af0b81 --- /dev/null +++ b/swh/storage/api/serializers.py @@ -0,0 +1,26 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +"""Decoder and encoders for swh-model objects.""" + +from typing import Callable, Dict, List, Tuple + +import swh.model.model as model + + +def _encode_model_object(obj): + d = obj.to_dict() + d['__type__'] = type(obj).__name__ + return d + + +ENCODERS: List[Tuple[type, str, Callable]] = [ + (model.BaseModel, 'model', _encode_model_object), +] + + +DECODERS: Dict[str, Callable] = { + 'model': lambda d: getattr(model, d.pop('__type__')).from_dict(d) +} diff --git a/swh/storage/api/server.py b/swh/storage/api/server.py index dd49c447..a16810d0 100644 --- a/swh/storage/api/server.py +++ b/swh/storage/api/server.py @@ -1,124 +1,131 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging from swh.core import config from swh.storage import get_storage as get_swhstorage from swh.core.api import (RPCServerApp, error_handler, encode_data_server as encode_data) from ..interface import StorageInterface from ..metrics import timed from ..exc import StorageArgumentException +from .serializers import ENCODERS, DECODERS + def get_storage(): global storage if not storage: storage = get_swhstorage(**app.config['storage']) return storage -app = RPCServerApp(__name__, - backend_class=StorageInterface, - backend_factory=get_storage) +class StorageServerApp(RPCServerApp): + extra_type_decoders = DECODERS + extra_type_encoders = ENCODERS + + +app = StorageServerApp(__name__, + backend_class=StorageInterface, + backend_factory=get_storage) storage = None @app.errorhandler(StorageArgumentException) def argument_error_handler(exception): return error_handler(exception, encode_data, status_code=400) @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.route('/') @timed def index(): return ''' Software Heritage storage server

You have reached the Software Heritage storage server.
See its documentation and API for more information

''' @app.route('/stat/counters', methods=['GET']) @timed def stat_counters(): return encode_data(get_storage().stat_counters()) @app.route('/stat/refresh', methods=['GET']) @timed def refresh_stat_counters(): return encode_data(get_storage().refresh_stat_counters()) api_cfg = None def load_and_check_config(config_file, type='local'): """Check the minimal configuration is set to run the api or raise an error explanation. Args: config_file (str): Path to the configuration file to load type (str): configuration type. For 'local' type, more checks are done. Raises: Error if the setup is not as expected Returns: configuration as a dict """ if not config_file: raise EnvironmentError('Configuration file must be defined') if not os.path.exists(config_file): raise FileNotFoundError('Configuration file %s does not exist' % ( config_file, )) cfg = config.read(config_file) if 'storage' not in cfg: raise KeyError("Missing '%storage' configuration") return cfg def make_app_from_configfile(): """Run the WSGI app from the webserver, loading the configuration from a configuration file. SWH_CONFIG_FILENAME environment variable defines the configuration path to load. """ global api_cfg if not api_cfg: config_file = os.environ.get('SWH_CONFIG_FILENAME') api_cfg = load_and_check_config(config_file) app.config.update(api_cfg) handler = logging.StreamHandler() app.logger.addHandler(handler) return app if __name__ == '__main__': print('Deprecated. Use swh-storage') diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client.py index 66123d22..eb42a538 100644 --- a/swh/storage/tests/test_api_client.py +++ b/swh/storage/tests/test_api_client.py @@ -1,69 +1,77 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from unittest.mock import patch import pytest -from swh.storage.api.client import RemoteStorage import swh.storage.api.server as server import swh.storage.storage +from swh.storage import get_storage from swh.storage.tests.test_storage import TestStorageGeneratedData # noqa from swh.storage.tests.test_storage import TestStorage as _TestStorage # tests are executed using imported classes (TestStorage and # TestStorageGeneratedData) using overloaded swh_storage fixture # below @pytest.fixture def app_server(): storage_config = { - 'cls': 'validate', - 'storage': { + 'cls': 'memory', + 'journal_writer': { 'cls': 'memory', - 'journal_writer': { - 'cls': 'memory', - }, - } + }, } server.storage = swh.storage.get_storage(**storage_config) yield server @pytest.fixture def app(app_server): return app_server.app @pytest.fixture def swh_rpc_client_class(): - return RemoteStorage + def storage_factory(**kwargs): + storage_config = { + 'cls': 'validate', + 'storage': { + 'cls': 'remote', + **kwargs, + } + } + return get_storage(**storage_config) + + return storage_factory @pytest.fixture def swh_storage(swh_rpc_client, app_server): # This version of the swh_storage fixture uses the swh_rpc_client fixture # to instantiate a RemoteStorage (see swh_rpc_client_class above) that # proxies, via the swh.core RPC mechanism, the local (in memory) storage # configured in the app_server fixture above. # # Also note that, for the sake of # making it easier to write tests, the in-memory journal writer of the # in-memory backend storage is attached to the RemoteStorage as its # journal_writer attribute. storage = swh_rpc_client + journal_writer = getattr(storage, 'journal_writer', None) storage.journal_writer = app_server.storage.journal_writer yield storage storage.journal_writer = journal_writer class TestStorage(_TestStorage): def test_content_update(self, swh_storage, app_server): # TODO, journal_writer not supported swh_storage.journal_writer.journal = None with patch.object(server.storage.journal_writer, 'journal', None): super().test_content_update(swh_storage) diff --git a/swh/storage/tests/test_api_client.py b/swh/storage/tests/test_api_client_dicts.py similarity index 100% copy from swh/storage/tests/test_api_client.py copy to swh/storage/tests/test_api_client_dicts.py