Differential D1108 Diff 3516 swh/vault/api/server.py

Changeset View

Standalone View

swh/vault/api/server.py

# Copyright (C) 2016-2018 The Software Heritage developers		# Copyright (C) 2016-2018 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU General Public License version 3, or any later version		# License: GNU General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

import aiohttp.web		import aiohttp.web
import asyncio		import asyncio
import click
import collections		import collections

from swh.core import config		from swh.core import config
from swh.core.api_async import (SWHRemoteAPI,		from swh.core.api_async import (SWHRemoteAPI,
encode_data_server as encode_data,		encode_data_server as encode_data,
decode_request)		decode_request)
from swh.model import hashutil		from swh.model import hashutil
		from swh.vault import get_vault
from swh.vault.cookers import COOKER_TYPES		from swh.vault.cookers import COOKER_TYPES
from swh.vault.backend import VaultBackend, NotFoundExc		from swh.vault.backend import NotFoundExc


DEFAULT_CONFIG_PATH = 'vault/server'		DEFAULT_CONFIG_PATH = 'vault/server'
DEFAULT_CONFIG = {		DEFAULT_CONFIG = {
'storage': ('dict', {		'storage': ('dict', {
'cls': 'remote',		'cls': 'remote',
'args': {		'args': {
'url': 'http://localhost:5002/',		'url': 'http://localhost:5002/',
},		},
}),		}),
'cache': ('dict', {		'cache': ('dict', {
'cls': 'pathslicing',		'cls': 'pathslicing',
'args': {		'args': {
'root': '/srv/softwareheritage/vault',		'root': '/srv/softwareheritage/vault',
'slicing': '0:1/1:5',		'slicing': '0:1/1:5',
},		},
}),		}),
'client_max_size': ('int', 1024 ** 3),		'client_max_size': ('int', 1024 ** 3),
'db': ('str', 'dbname=softwareheritage-vault-dev'),		'vault': ('dict', {
		'cls': 'local',
		'args': {
		'db': 'dbname=softwareheritage-vault-dev',
		},
		}),
'scheduler': ('dict', {		'scheduler': ('dict', {
'cls': 'remote',		'cls': 'remote',
'args': {		'args': {
'url': 'http://localhost:5008/',		'url': 'http://localhost:5008/',
}		}
}),		}),
}		}

▲ Show 20 Lines • Show All 118 Lines • ▼ Show 20 Lines	def batch_progress(request):
res = {'bundles': bundles, 'total': len(bundles),		res = {'bundles': bundles, 'total': len(bundles),
**{k: 0 for k in ('new', 'pending', 'done', 'failed')},		**{k: 0 for k in ('new', 'pending', 'done', 'failed')},
**dict(counter)}		**dict(counter)}
return encode_data(res)		return encode_data(res)


# Web server		# Web server

def make_app(config, **kwargs):		def make_app(backend, **kwargs):
if 'client_max_size' in config:
kwargs['client_max_size'] = config['client_max_size']

app = SWHRemoteAPI(**kwargs)		app = SWHRemoteAPI(**kwargs)
app.router.add_route('GET', '/', index)		app.router.add_route('GET', '/', index)

# Endpoints used by the web API		# Endpoints used by the web API
app.router.add_route('GET', '/fetch/{type}/{id}', vault_fetch)		app.router.add_route('GET', '/fetch/{type}/{id}', vault_fetch)
app.router.add_route('POST', '/cook/{type}/{id}', vault_cook)		app.router.add_route('POST', '/cook/{type}/{id}', vault_cook)
app.router.add_route('GET', '/progress/{type}/{id}', vault_progress)		app.router.add_route('GET', '/progress/{type}/{id}', vault_progress)

# Endpoints used by the Cookers		# Endpoints used by the Cookers
app.router.add_route('POST', '/set_progress/{type}/{id}', set_progress)		app.router.add_route('POST', '/set_progress/{type}/{id}', set_progress)
app.router.add_route('POST', '/set_status/{type}/{id}', set_status)		app.router.add_route('POST', '/set_status/{type}/{id}', set_status)
app.router.add_route('POST', '/put_bundle/{type}/{id}', put_bundle)		app.router.add_route('POST', '/put_bundle/{type}/{id}', put_bundle)
app.router.add_route('POST', '/send_notif/{type}/{id}', send_notif)		app.router.add_route('POST', '/send_notif/{type}/{id}', send_notif)

# Endpoints for batch requests		# Endpoints for batch requests
app.router.add_route('POST', '/batch_cook', batch_cook)		app.router.add_route('POST', '/batch_cook', batch_cook)
app.router.add_route('GET', '/batch_progress/{batch_id}', batch_progress)		app.router.add_route('GET', '/batch_progress/{batch_id}', batch_progress)

app['backend'] = VaultBackend(config)		app['backend'] = backend
return app		return app


def make_app_from_configfile(config_path=DEFAULT_CONFIG_PATH, **kwargs):		def get_local_backend(config_file):
cfg = config.load_named_config(config_path, DEFAULT_CONFIG)		cfg = config.read(config_file, DEFAULT_CONFIG)
return make_app(cfg, **kwargs)		if 'vault' not in cfg:
		raise ValueError("missing '%vault' configuration")

@click.command()		vcfg = cfg['vault']
@click.argument('config-path', required=1)		if vcfg['cls'] != 'local':
@click.option('--host', default='0.0.0.0', help="Host to run the server")		raise EnvironmentError(
@click.option('--port', default=5005, type=click.INT,		"The vault backend can only be started with a 'local' "
help="Binding port of the server")		"configuration", err=True)
@click.option('--debug/--nodebug', default=True,		args = vcfg['args']
help="Indicates if the server should run in debug mode")		if 'cache' not in args:
def launch(config_path, host, port, debug):		args['cache'] = cfg.get('cache')
app = make_app(config.read(config_path, DEFAULT_CONFIG), debug=bool(debug))		if 'storage' not in args:
aiohttp.web.run_app(app, host=host, port=int(port))		args['storage'] = cfg.get('storage')
		if 'scheduler' not in args:
		args['scheduler'] = cfg.get('scheduler')

		for key in ('cache', 'storage', 'scheduler'):
		if not args.get(key):
		raise ValueError(
		"invalid configuration; missing %s config entry." % key)

		return get_vault('local', args)


		def make_app_from_configfile(config_file, **kwargs):
		vault = get_local_backend(config_file)
		return make_app(backend=vault, **kwargs)


if __name__ == '__main__':		if __name__ == '__main__':
launch()		print('Deprecated. Use swh-vault ')