Changeset View
Changeset View
Standalone View
Standalone View
swh/vault/api/server.py
# Copyright (C) 2016-2018 The Software Heritage developers | # Copyright (C) 2016-2018 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
import aiohttp.web | import aiohttp.web | ||||
import asyncio | import asyncio | ||||
import click | |||||
import collections | import collections | ||||
from swh.core import config | from swh.core import config | ||||
from swh.core.api_async import (SWHRemoteAPI, | from swh.core.api_async import (SWHRemoteAPI, | ||||
encode_data_server as encode_data, | encode_data_server as encode_data, | ||||
decode_request) | decode_request) | ||||
from swh.model import hashutil | from swh.model import hashutil | ||||
from swh.vault import get_vault | |||||
from swh.vault.cookers import COOKER_TYPES | from swh.vault.cookers import COOKER_TYPES | ||||
from swh.vault.backend import VaultBackend, NotFoundExc | from swh.vault.backend import NotFoundExc | ||||
DEFAULT_CONFIG_PATH = 'vault/server' | DEFAULT_CONFIG_PATH = 'vault/server' | ||||
DEFAULT_CONFIG = { | DEFAULT_CONFIG = { | ||||
'storage': ('dict', { | 'storage': ('dict', { | ||||
'cls': 'remote', | 'cls': 'remote', | ||||
'args': { | 'args': { | ||||
'url': 'http://localhost:5002/', | 'url': 'http://localhost:5002/', | ||||
}, | }, | ||||
}), | }), | ||||
'cache': ('dict', { | 'cache': ('dict', { | ||||
'cls': 'pathslicing', | 'cls': 'pathslicing', | ||||
'args': { | 'args': { | ||||
'root': '/srv/softwareheritage/vault', | 'root': '/srv/softwareheritage/vault', | ||||
'slicing': '0:1/1:5', | 'slicing': '0:1/1:5', | ||||
}, | }, | ||||
}), | }), | ||||
'client_max_size': ('int', 1024 ** 3), | 'client_max_size': ('int', 1024 ** 3), | ||||
'db': ('str', 'dbname=softwareheritage-vault-dev'), | 'vault': ('dict', { | ||||
'cls': 'local', | |||||
'args': { | |||||
'db': 'dbname=softwareheritage-vault-dev', | |||||
}, | |||||
}), | |||||
'scheduler': ('dict', { | 'scheduler': ('dict', { | ||||
'cls': 'remote', | 'cls': 'remote', | ||||
'args': { | 'args': { | ||||
'url': 'http://localhost:5008/', | 'url': 'http://localhost:5008/', | ||||
} | } | ||||
}), | }), | ||||
} | } | ||||
▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines | def batch_progress(request): | ||||
res = {'bundles': bundles, 'total': len(bundles), | res = {'bundles': bundles, 'total': len(bundles), | ||||
**{k: 0 for k in ('new', 'pending', 'done', 'failed')}, | **{k: 0 for k in ('new', 'pending', 'done', 'failed')}, | ||||
**dict(counter)} | **dict(counter)} | ||||
return encode_data(res) | return encode_data(res) | ||||
# Web server | # Web server | ||||
def make_app(config, **kwargs): | def make_app(backend, **kwargs): | ||||
if 'client_max_size' in config: | |||||
kwargs['client_max_size'] = config['client_max_size'] | |||||
app = SWHRemoteAPI(**kwargs) | app = SWHRemoteAPI(**kwargs) | ||||
app.router.add_route('GET', '/', index) | app.router.add_route('GET', '/', index) | ||||
# Endpoints used by the web API | # Endpoints used by the web API | ||||
app.router.add_route('GET', '/fetch/{type}/{id}', vault_fetch) | app.router.add_route('GET', '/fetch/{type}/{id}', vault_fetch) | ||||
app.router.add_route('POST', '/cook/{type}/{id}', vault_cook) | app.router.add_route('POST', '/cook/{type}/{id}', vault_cook) | ||||
app.router.add_route('GET', '/progress/{type}/{id}', vault_progress) | app.router.add_route('GET', '/progress/{type}/{id}', vault_progress) | ||||
# Endpoints used by the Cookers | # Endpoints used by the Cookers | ||||
app.router.add_route('POST', '/set_progress/{type}/{id}', set_progress) | app.router.add_route('POST', '/set_progress/{type}/{id}', set_progress) | ||||
app.router.add_route('POST', '/set_status/{type}/{id}', set_status) | app.router.add_route('POST', '/set_status/{type}/{id}', set_status) | ||||
app.router.add_route('POST', '/put_bundle/{type}/{id}', put_bundle) | app.router.add_route('POST', '/put_bundle/{type}/{id}', put_bundle) | ||||
app.router.add_route('POST', '/send_notif/{type}/{id}', send_notif) | app.router.add_route('POST', '/send_notif/{type}/{id}', send_notif) | ||||
# Endpoints for batch requests | # Endpoints for batch requests | ||||
app.router.add_route('POST', '/batch_cook', batch_cook) | app.router.add_route('POST', '/batch_cook', batch_cook) | ||||
app.router.add_route('GET', '/batch_progress/{batch_id}', batch_progress) | app.router.add_route('GET', '/batch_progress/{batch_id}', batch_progress) | ||||
app['backend'] = VaultBackend(config) | app['backend'] = backend | ||||
return app | return app | ||||
def make_app_from_configfile(config_path=DEFAULT_CONFIG_PATH, **kwargs): | def get_local_backend(config_file): | ||||
cfg = config.load_named_config(config_path, DEFAULT_CONFIG) | cfg = config.read(config_file, DEFAULT_CONFIG) | ||||
return make_app(cfg, **kwargs) | if 'vault' not in cfg: | ||||
raise ValueError("missing '%vault' configuration") | |||||
@click.command() | vcfg = cfg['vault'] | ||||
@click.argument('config-path', required=1) | if vcfg['cls'] != 'local': | ||||
@click.option('--host', default='0.0.0.0', help="Host to run the server") | raise EnvironmentError( | ||||
@click.option('--port', default=5005, type=click.INT, | "The vault backend can only be started with a 'local' " | ||||
help="Binding port of the server") | "configuration", err=True) | ||||
@click.option('--debug/--nodebug', default=True, | args = vcfg['args'] | ||||
help="Indicates if the server should run in debug mode") | if 'cache' not in args: | ||||
def launch(config_path, host, port, debug): | args['cache'] = cfg.get('cache') | ||||
app = make_app(config.read(config_path, DEFAULT_CONFIG), debug=bool(debug)) | if 'storage' not in args: | ||||
aiohttp.web.run_app(app, host=host, port=int(port)) | args['storage'] = cfg.get('storage') | ||||
if 'scheduler' not in args: | |||||
args['scheduler'] = cfg.get('scheduler') | |||||
for key in ('cache', 'storage', 'scheduler'): | |||||
if not args.get(key): | |||||
raise ValueError( | |||||
"invalid configuration; missing %s config entry." % key) | |||||
return get_vault('local', args) | |||||
def make_app_from_configfile(config_file, **kwargs): | |||||
vault = get_local_backend(config_file) | |||||
return make_app(backend=vault, **kwargs) | |||||
if __name__ == '__main__': | if __name__ == '__main__': | ||||
launch() | print('Deprecated. Use swh-vault ') |