diff --git a/swh/deposit/backend.py b/swh/deposit/backend.py index 8647642c..68a1e739 100644 --- a/swh/deposit/backend.py +++ b/swh/deposit/backend.py @@ -1,163 +1,152 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from functools import wraps import psycopg2 import psycopg2.extras -from swh.core.config import SWHConfig - psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) def autocommit(fn): @wraps(fn) def wrapped(self, *args, **kwargs): autocommit = False if 'cursor' not in kwargs or not kwargs['cursor']: autocommit = True kwargs['cursor'] = self.cursor() try: ret = fn(self, *args, **kwargs) except: if autocommit: self.rollback() raise if autocommit: self.commit() return ret return wrapped -class DepositBackend(SWHConfig): +class DepositBackend(): """Backend for the Software Heritage deposit database. """ - CONFIG_BASE_FILENAME = 'deposit/backend' - - DEFAULT_CONFIG = { - 'deposit_db': ('str', 'dbname=swh-deposit'), - } - - def __init__(self, **override_config): - self.config = self.parse_config_file(global_config=False) - self.config.update(override_config) - + def __init__(self, dbconn): self.db = None - + self.dbconn = dbconn self.reconnect() def reconnect(self): if not self.db or self.db.closed: self.db = psycopg2.connect( - dsn=self.config['deposit_db'], + dsn=self.dbconn, cursor_factory=psycopg2.extras.RealDictCursor, ) def cursor(self): """Return a fresh cursor on the database, with auto-reconnection in case of failure """ cur = None # Get a fresh cursor and reconnect at most three times tries = 0 while True: tries += 1 try: cur = self.db.cursor() cur.execute('select 1') break except psycopg2.OperationalError: if tries < 3: self.reconnect() else: raise return cur def commit(self): """Commit a transaction""" self.db.commit() def rollback(self): """Rollback a transaction""" self.db.rollback() deposit_keys = [ 'reception_date', 'complete_date', 'type', 'external_id', 'status', 'client_id', ] def _format_query(self, query, keys): """Format a query with the given keys""" query_keys = ', '.join(keys) placeholders = ', '.join(['%s'] * len(keys)) return query.format(keys=query_keys, placeholders=placeholders) @autocommit def deposit_add(self, deposit, cursor=None): """Create a new deposit. A deposit is a dictionary with the following keys: type (str): an identifier for the deposit type reception_date (date): deposit's reception date complete_date (date): deposit's date when the deposit is deemed complete external_id (str): the external identifier in the client's information system status (str): deposit status client_id (integer): client's identifier """ query = self._format_query( """insert into deposit ({keys}) values ({placeholders})""", self.deposit_keys, ) cursor.execute(query, [deposit[key] for key in self.deposi_keys]) @autocommit def deposit_get(self, id, cursor=None): """Retrieve the task type with id """ query = self._format_query( "select {keys} from deposit where type=%s", self.deposit_keys, ) cursor.execute(query, (id,)) ret = cursor.fetchone() return ret @autocommit def request_add(self, request, cursor=None): pass @autocommit def request_get(self, deposit_id, cursor=None): pass @autocommit def client_list(self, cursor=None): cursor.execute('select id, name from client') return {row['name']: row['id'] for row in cursor.fetchall()} @autocommit def client_get(self, id, cursor=None): cursor.execute('select id, name, credential from client where id=%s', (id, )) return cursor.fetchone() diff --git a/swh/deposit/server.py b/swh/deposit/server.py index 924d2fae..c2634161 100644 --- a/swh/deposit/server.py +++ b/swh/deposit/server.py @@ -1,133 +1,133 @@ # Copyright (C) 2017 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import asyncio import aiohttp.web import click import jinja2 import json from swh.core import config from swh.core.config import SWHConfig from swh.core.api_async import SWHRemoteAPI from swh.deposit.backend import DepositBackend DEFAULT_CONFIG_PATH = 'deposit/server' DEFAULT_CONFIG = { 'host': ('str', '0.0.0.0'), 'port': ('int', 5006), } def encode_data(data, template_name=None, **kwargs): return aiohttp.web.Response( body=data, headers={'Content-Type': 'application/xml'}, **kwargs ) class DepositWebServer(SWHConfig): """Base class to define endpoints route. """ CONFIG_BASE_FILENAME = DEFAULT_CONFIG_PATH DEFAULT_CONFIG = { 'max_upload_size': ('int', 209715200), - 'deposit_db': ('str', 'dbname=softwareheritage-deposit-dev'), + 'dbconn': ('str', 'dbname=softwareheritage-deposit-dev'), } def __init__(self, config=None): if config: self.config = config else: self.config = self.parse_config_file() template_loader = jinja2.FileSystemLoader( searchpath=["swh/deposit/templates"]) self.template_env = jinja2.Environment(loader=template_loader) - self.backend = DepositBackend() + self.backend = DepositBackend(self.config['dbconn']) @asyncio.coroutine def index(self, request): return aiohttp.web.Response(text='SWH Deposit Server') @asyncio.coroutine def service_document(self, request): tpl = self.template_env.get_template('service_document.xml') output = tpl.render( noop=True, verbose=False, max_upload_size=self.config['max_upload_size']) return encode_data(data=output) @asyncio.coroutine def create_document(self, request): pass @asyncio.coroutine def update_document(self, request): pass @asyncio.coroutine def status_operation(self, request): pass @asyncio.coroutine def delete_document(self, request): raise ValueError('Not implemented') @asyncio.coroutine def client_get(self, request): clients = self.backend.client_list() return aiohttp.web.Response( body=json.dumps(clients), headers={'Content-Type': 'application/json'}) def make_app(config, **kwargs): """Initialize server application. Returns: Application ready for running and serving api endpoints. """ app = SWHRemoteAPI(**kwargs) server = DepositWebServer() app.router.add_route('GET', '/', server.index) app.router.add_route('GET', '/api/1/deposit/', server.service_document) app.router.add_route('GET', '/api/1/status/', server.status_operation) app.router.add_route('POST', '/api/1/deposit/', server.create_document) app.router.add_route('PUT', '/api/1/deposit/', server.update_document) app.router.add_route('DELETE', '/api/1/deposit/', server.delete_document) app.router.add_route('GET', '/api/1/client/', server.client_get) app.update(config) return app def make_app_from_configfile(config_path=DEFAULT_CONFIG_PATH, **kwargs): """Initialize server application from configuration file. Returns: Application ready for running and serving api endpoints. """ return make_app(config.read(config_path, DEFAULT_CONFIG), **kwargs) @click.command() @click.argument('config-path', required=1) @click.option('--host', default='0.0.0.0', help="Host to run the server") @click.option('--port', default=5006, type=click.INT, help="Binding port of the server") @click.option('--debug/--nodebug', default=True, help="Indicates if the server should run in debug mode") def launch(config_path, host, port, debug): app = make_app_from_configfile(config_path, debug=bool(debug)) aiohttp.web.run_app(app, host=host, port=port) if __name__ == '__main__': launch()