diff --git a/PKG-INFO b/PKG-INFO index 2a00b29..7003604 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.16 +Version: 0.0.17 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/debian/control b/debian/control index 242fd04..e84cecb 100644 --- a/debian/control +++ b/debian/control @@ -1,38 +1,37 @@ Source: swh-objstorage Maintainer: Software Heritage developers Section: python Priority: optional Build-Depends: debhelper (>= 9), dh-python, python3-all, python3-flask, python3-nose, - python3-requests, python3-setuptools, - python3-swh.core (>= 0.0.20~), + python3-swh.core (>= 0.0.28~), python3-swh.storage.archiver (>= 0.0.52~), python3-click, python3-libcloud, python3-azure-storage, python3-vcversioner Standards-Version: 3.9.6 Homepage: https://forge.softwareheritage.org/diffusion/DOBJS/ Package: python3-swh.objstorage Architecture: all -Depends: python3-swh.core (>= 0.0.20~), ${misc:Depends}, ${python3:Depends} +Depends: python3-swh.core (>= 0.0.28~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage Object Storage Package: python3-swh.objstorage.checker Architecture: all Depends: python3-swh.objstorage (= ${binary:Version}), python3-swh.storage.archiver (>= 0.0.52~), ${misc:Depends}, ${python3:Depends} Description: Software Heritage Object Storage Checker Package: python3-swh.objstorage.cloud Architecture: all Depends: python3-swh.objstorage (= ${binary:Version}), python3-libcloud, python3-azure-storage, ${misc:Depends}, ${python3:Depends} Breaks: python3-swh.objstorage (<= 0.0.7~) Description: Software Heritage Cloud Object Storage diff --git a/requirements.txt b/requirements.txt index a7f611f..0ad5d4e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,19 +1,16 @@ # Add here external Python modules dependencies, one per line. Module names # should match https://pypi.python.org/pypi names. For the full spec or # dependency lines, see https://pip.readthedocs.org/en/1.1/requirements.html vcversioner -# remote storage API client -requests - # remote storage API server flask # Internal dependencies -swh.core >= 0.0.20 +swh.core >= 0.0.28 swh.storage.archiver >= 0.0.52 click # optional dependencies # apache-libcloud # azure-storage diff --git a/swh.objstorage.egg-info/PKG-INFO b/swh.objstorage.egg-info/PKG-INFO index 2a00b29..7003604 100644 --- a/swh.objstorage.egg-info/PKG-INFO +++ b/swh.objstorage.egg-info/PKG-INFO @@ -1,10 +1,10 @@ Metadata-Version: 1.0 Name: swh.objstorage -Version: 0.0.16 +Version: 0.0.17 Summary: Software Heritage Object Storage Home-page: https://forge.softwareheritage.org/diffusion/DOBJS Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN diff --git a/swh.objstorage.egg-info/SOURCES.txt b/swh.objstorage.egg-info/SOURCES.txt index 4651671..8e3e568 100644 --- a/swh.objstorage.egg-info/SOURCES.txt +++ b/swh.objstorage.egg-info/SOURCES.txt @@ -1,50 +1,49 @@ .gitignore AUTHORS LICENSE MANIFEST.in Makefile requirements.txt setup.py version.txt bin/swh-objstorage-add-dir bin/swh-objstorage-azure bin/swh-objstorage-fsck debian/changelog debian/compat debian/control debian/copyright debian/rules debian/source/format swh.objstorage.egg-info/PKG-INFO swh.objstorage.egg-info/SOURCES.txt swh.objstorage.egg-info/dependency_links.txt swh.objstorage.egg-info/requires.txt swh.objstorage.egg-info/top_level.txt swh/objstorage/__init__.py swh/objstorage/checker.py swh/objstorage/exc.py swh/objstorage/objstorage.py swh/objstorage/objstorage_pathslicing.py swh/objstorage/api/__init__.py swh/objstorage/api/client.py -swh/objstorage/api/common.py swh/objstorage/api/server.py swh/objstorage/cloud/__init__.py swh/objstorage/cloud/objstorage_azure.py swh/objstorage/cloud/objstorage_cloud.py swh/objstorage/multiplexer/__init__.py swh/objstorage/multiplexer/multiplexer_objstorage.py swh/objstorage/multiplexer/filter/__init__.py swh/objstorage/multiplexer/filter/filter.py swh/objstorage/multiplexer/filter/id_filter.py swh/objstorage/multiplexer/filter/read_write_filter.py swh/objstorage/tests/objstorage_testing.py swh/objstorage/tests/server_testing.py swh/objstorage/tests/test_checker.py swh/objstorage/tests/test_multiplexer_filter.py swh/objstorage/tests/test_objstorage_api.py swh/objstorage/tests/test_objstorage_azure.py swh/objstorage/tests/test_objstorage_cloud.py swh/objstorage/tests/test_objstorage_instantiation.py swh/objstorage/tests/test_objstorage_multiplexer.py swh/objstorage/tests/test_objstorage_pathslicing.py \ No newline at end of file diff --git a/swh.objstorage.egg-info/requires.txt b/swh.objstorage.egg-info/requires.txt index 921badb..e596870 100644 --- a/swh.objstorage.egg-info/requires.txt +++ b/swh.objstorage.egg-info/requires.txt @@ -1,6 +1,5 @@ click flask -requests -swh.core>=0.0.20 +swh.core>=0.0.28 swh.storage.archiver>=0.0.52 vcversioner diff --git a/swh/objstorage/api/client.py b/swh/objstorage/api/client.py index 437e4a5..b359931 100644 --- a/swh/objstorage/api/client.py +++ b/swh/objstorage/api/client.py @@ -1,72 +1,48 @@ -# Copyright (C) 2015-2016 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import pickle -import requests +from swh.core.api import SWHRemoteAPI -from requests.exceptions import ConnectionError from ..objstorage import ObjStorage from ..exc import ObjStorageAPIError -from .common import (decode_response, - encode_data_client as encode_data) -class RemoteObjStorage(ObjStorage): - """ Proxy to a remote object storage. +class RemoteObjStorage(ObjStorage, SWHRemoteAPI): + """Proxy to a remote object storage. This class allows to connect to an object storage server via http protocol. Attributes: url (string): The url of the server to connect. Must end with a '/' session: The session to send requests. + """ def __init__(self, url): - self.url = url - self.session = requests.Session() - - def _url(self, endpoint): - return '%s%s' % (self.url, endpoint) - - def post(self, endpoint, data): - try: - response = self.session.post( - self._url(endpoint), - data=encode_data(data), - headers={'content-type': 'application/x-msgpack'}, - ) - except ConnectionError as e: - raise ObjStorageAPIError(e) - - # XXX: this breaks language-independence and should be - # replaced by proper unserialization - if response.status_code == 400: - raise pickle.loads(decode_response(response)) - - return decode_response(response) + super().__init__(api_exception=ObjStorageAPIError, url=url) def check_config(self, *, check_write): return self.post('check_config', {'check_write': check_write}) def __contains__(self, obj_id): return self.post('content/contains', {'obj_id': obj_id}) def add(self, content, obj_id=None, check_presence=True): return self.post('content/add', {'content': content, 'obj_id': obj_id, 'check_presence': check_presence}) def get(self, obj_id): return self.post('content/get', {'obj_id': obj_id}) def get_batch(self, obj_ids): return self.post('content/get/batch', {'obj_ids': obj_ids}) def check(self, obj_id): self.post('content/check', {'obj_id': obj_id}) def get_random(self, batch_size): return self.post('content/get/random', {'batch_size': batch_size}) diff --git a/swh/objstorage/api/common.py b/swh/objstorage/api/common.py deleted file mode 100644 index 328d826..0000000 --- a/swh/objstorage/api/common.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (C) 2015 The Software Heritage developers -# See the AUTHORS file at the top-level directory of this distribution -# License: GNU General Public License version 3, or any later version -# See top-level LICENSE file for more information - -import json -import pickle - -from flask import Request, Response - -from swh.core.serializers import msgpack_dumps, msgpack_loads, SWHJSONDecoder - - -class BytesRequest(Request): - """Request with proper escaping of arbitrary byte sequences.""" - encoding = 'utf-8' - encoding_errors = 'surrogateescape' - - -def encode_data_server(data): - return Response( - msgpack_dumps(data), - mimetype='application/x-msgpack', - ) - - -def encode_data_client(data): - try: - return msgpack_dumps(data) - except OverflowError as e: - raise ValueError('Limits were reached. Please, check your input.\n' + - str(e)) - - -def decode_request(request): - content_type = request.mimetype - data = request.get_data() - - if content_type == 'application/x-msgpack': - r = msgpack_loads(data) - elif content_type == 'application/json': - r = json.loads(data, cls=SWHJSONDecoder) - else: - raise ValueError('Wrong content type `%s` for API request' - % content_type) - - return r - - -def decode_response(response): - content_type = response.headers['content-type'] - - if content_type.startswith('application/x-msgpack'): - r = msgpack_loads(response.content) - elif content_type.startswith('application/json'): - r = response.json(cls=SWHJSONDecoder) - else: - raise ValueError('Wrong content type `%s` for API response' - % content_type) - - return r - - -def error_handler(exception, encoder): - # XXX: this breaks language-independence and should be - # replaced by proper serialization of errors - response = encoder(pickle.dumps(exception)) - response.status_code = 400 - return response diff --git a/swh/objstorage/api/server.py b/swh/objstorage/api/server.py index 2e983e9..3c4c063 100644 --- a/swh/objstorage/api/server.py +++ b/swh/objstorage/api/server.py @@ -1,114 +1,113 @@ -# Copyright (C) 2015 The Software Heritage developers +# Copyright (C) 2015-2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import click import logging -from flask import Flask, g, request +from flask import g, request from swh.core import config +from swh.core.api import (SWHServerAPIApp, decode_request, + error_handler, + encode_data_server as encode_data) from swh.objstorage import get_objstorage -from swh.objstorage.api.common import (BytesRequest, decode_request, - error_handler, - encode_data_server as encode_data) DEFAULT_CONFIG = { 'cls': ('str', 'pathslicing'), 'args': ('dict', { 'root': '/srv/softwareheritage/objects', 'slicing': '0:2/2:4/4:6', }) } -app = Flask(__name__) -app.request_class = BytesRequest +app = SWHServerAPIApp(__name__) @app.errorhandler(Exception) def my_error_handler(exception): return error_handler(exception, encode_data) @app.before_request def before_request(): g.objstorage = get_objstorage(app.config['cls'], app.config['args']) @app.route('/') def index(): return "SWH Objstorage API server" @app.route('/check_config', methods=['POST']) def check_config(): return encode_data(g.objstorage.check_config(**decode_request(request))) @app.route('/content') def content(): return str(list(g.storage)) @app.route('/content/contains', methods=['POST']) def contains(): return encode_data(g.objstorage.__contains__(**decode_request(request))) @app.route('/content/add', methods=['POST']) def add_bytes(): return encode_data(g.objstorage.add(**decode_request(request))) @app.route('/content/get', methods=['POST']) def get_bytes(): return encode_data(g.objstorage.get(**decode_request(request))) @app.route('/content/get/batch', methods=['POST']) def get_batch(): return encode_data(g.objstorage.get_batch(**decode_request(request))) @app.route('/content/get/random', methods=['POST']) def get_random_contents(): return encode_data( g.objstorage.get_random(**decode_request(request)) ) @app.route('/content/check', methods=['POST']) def check(): return encode_data(g.objstorage.check(**decode_request(request))) def run_from_webserver(environ, start_response): """Run the WSGI app from the webserver, loading the configuration. """ config_path = '/etc/softwareheritage/storage/objstorage.yml' app.config.update(config.read(config_path, DEFAULT_CONFIG)) handler = logging.StreamHandler() app.logger.addHandler(handler) return app(environ, start_response) @click.command() @click.argument('config-path', required=1) @click.option('--host', default='0.0.0.0', help="Host to run the server") @click.option('--port', default=5000, type=click.INT, help="Binding port of the server") @click.option('--debug/--nodebug', default=True, help="Indicates if the server should run in debug mode") def launch(config_path, host, port, debug): app.config.update(config.read(config_path, DEFAULT_CONFIG)) app.run(host, port=int(port), debug=bool(debug)) if __name__ == '__main__': launch() diff --git a/version.txt b/version.txt index c4d9ecb..5113053 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -v0.0.16-0-g39db1d6 \ No newline at end of file +v0.0.17-0-gcf8a95b \ No newline at end of file