diff --git a/setup.py b/setup.py index 7efe02b..2762119 100755 --- a/setup.py +++ b/setup.py @@ -1,65 +1,69 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.vault', description='Software Heritage vault', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DVAU/', packages=find_packages(), install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, zip_safe=False, + entry_points=''' + [console_scripts] + swh-vault=swh.vault.cli:main + ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-vault', }, ) diff --git a/swh/vault/api/server.py b/swh/vault/api/server.py index cc9665b..5894e9d 100644 --- a/swh/vault/api/server.py +++ b/swh/vault/api/server.py @@ -1,220 +1,207 @@ # Copyright (C) 2016-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import aiohttp.web import asyncio -import click import collections from swh.core import config from swh.core.api_async import (SWHRemoteAPI, encode_data_server as encode_data, decode_request) from swh.model import hashutil from swh.vault.cookers import COOKER_TYPES from swh.vault.backend import VaultBackend, NotFoundExc DEFAULT_CONFIG_PATH = 'vault/server' DEFAULT_CONFIG = { 'storage': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://localhost:5002/', }, }), 'cache': ('dict', { 'cls': 'pathslicing', 'args': { 'root': '/srv/softwareheritage/vault', 'slicing': '0:1/1:5', }, }), 'client_max_size': ('int', 1024 ** 3), 'db': ('str', 'dbname=softwareheritage-vault-dev'), 'scheduler': ('dict', { 'cls': 'remote', 'args': { 'url': 'http://localhost:5008/', } }), } @asyncio.coroutine def index(request): return aiohttp.web.Response(body="SWH Vault API server") # Web API endpoints @asyncio.coroutine def vault_fetch(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] if not request.app['backend'].is_available(obj_type, obj_id): raise aiohttp.web.HTTPNotFound return encode_data(request.app['backend'].fetch(obj_type, obj_id)) def user_info(task_info): return {'id': task_info['id'], 'status': task_info['task_status'], 'progress_message': task_info['progress_msg'], 'obj_type': task_info['type'], 'obj_id': hashutil.hash_to_hex(task_info['object_id'])} @asyncio.coroutine def vault_cook(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] email = request.query.get('email') sticky = request.query.get('sticky') in ('true', '1') if obj_type not in COOKER_TYPES: raise aiohttp.web.HTTPNotFound try: info = request.app['backend'].cook_request(obj_type, obj_id, email=email, sticky=sticky) except NotFoundExc: raise aiohttp.web.HTTPNotFound # TODO: return 201 status (Created) once the api supports it return encode_data(user_info(info)) @asyncio.coroutine def vault_progress(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] info = request.app['backend'].task_info(obj_type, obj_id) if not info: raise aiohttp.web.HTTPNotFound return encode_data(user_info(info)) # Cookers endpoints @asyncio.coroutine def set_progress(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] progress = yield from decode_request(request) request.app['backend'].set_progress(obj_type, obj_id, progress) return encode_data(True) # FIXME: success value? @asyncio.coroutine def set_status(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] status = yield from decode_request(request) request.app['backend'].set_status(obj_type, obj_id, status) return encode_data(True) # FIXME: success value? @asyncio.coroutine def put_bundle(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] # TODO: handle streaming properly content = yield from decode_request(request) request.app['backend'].cache.add(obj_type, obj_id, content) return encode_data(True) # FIXME: success value? @asyncio.coroutine def send_notif(request): obj_type = request.match_info['type'] obj_id = request.match_info['id'] request.app['backend'].send_all_notifications(obj_type, obj_id) return encode_data(True) # FIXME: success value? # Batch endpoints @asyncio.coroutine def batch_cook(request): batch = yield from decode_request(request) for obj_type, obj_id in batch: if obj_type not in COOKER_TYPES: raise aiohttp.web.HTTPNotFound batch_id = request.app['backend'].batch_cook(batch) return encode_data({'id': batch_id}) @asyncio.coroutine def batch_progress(request): batch_id = request.match_info['batch_id'] bundles = request.app['backend'].batch_info(batch_id) if not bundles: raise aiohttp.web.HTTPNotFound bundles = [user_info(bundle) for bundle in bundles] counter = collections.Counter(b['status'] for b in bundles) res = {'bundles': bundles, 'total': len(bundles), **{k: 0 for k in ('new', 'pending', 'done', 'failed')}, **dict(counter)} return encode_data(res) # Web server def make_app(config, **kwargs): if 'client_max_size' in config: kwargs['client_max_size'] = config['client_max_size'] app = SWHRemoteAPI(**kwargs) app.router.add_route('GET', '/', index) # Endpoints used by the web API app.router.add_route('GET', '/fetch/{type}/{id}', vault_fetch) app.router.add_route('POST', '/cook/{type}/{id}', vault_cook) app.router.add_route('GET', '/progress/{type}/{id}', vault_progress) # Endpoints used by the Cookers app.router.add_route('POST', '/set_progress/{type}/{id}', set_progress) app.router.add_route('POST', '/set_status/{type}/{id}', set_status) app.router.add_route('POST', '/put_bundle/{type}/{id}', put_bundle) app.router.add_route('POST', '/send_notif/{type}/{id}', send_notif) # Endpoints for batch requests app.router.add_route('POST', '/batch_cook', batch_cook) app.router.add_route('GET', '/batch_progress/{batch_id}', batch_progress) app['backend'] = VaultBackend(config) return app api_cfg = None def make_app_from_configfile(config_path=DEFAULT_CONFIG_PATH, **kwargs): global api_cfg if not api_cfg: api_cfg = config.read(config_path, DEFAULT_CONFIG) return make_app(api_cfg, **kwargs) -@click.command() -@click.argument('config-path', required=1) -@click.option('--host', default='0.0.0.0', help="Host to run the server") -@click.option('--port', default=5005, type=click.INT, - help="Binding port of the server") -@click.option('--debug/--nodebug', default=True, - help="Indicates if the server should run in debug mode") -def launch(config_path, host, port, debug): - app = make_app(config.read(config_path, DEFAULT_CONFIG), debug=bool(debug)) - aiohttp.web.run_app(app, host=host, port=int(port)) - - if __name__ == '__main__': - launch() + print('Deprecated. Use swh-vault ') diff --git a/swh/vault/cli.py b/swh/vault/cli.py new file mode 100644 index 0000000..3037962 --- /dev/null +++ b/swh/vault/cli.py @@ -0,0 +1,91 @@ +import logging +import click +import aiohttp + +from swh.core import config +from swh.vault import get_vault +from swh.vault.api.server import make_app, DEFAULT_CONFIG + + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS) +@click.option('--config-file', '-C', default=None, + type=click.Path(exists=True, dir_okay=False,), + help="Configuration file.") +@click.option('--database', '-d', default=None, + help="Scheduling database DSN (imply cls is 'local')") +@click.option('--url', '-u', default=None, + help="Scheduler's url access (imply cls is 'remote')") +@click.option('--log-level', '-l', default='INFO', + type=click.Choice(logging._nameToLevel.keys()), + help="Log level (default to INFO)") +@click.option('--no-stdout', is_flag=True, default=False, + help="Do NOT output logs on the console") +@click.pass_context +def cli(ctx, config_file, database, url, log_level, no_stdout): + """Software Heritage Scheduler CLI interface + + Default to use the the local scheduler instance (plugged to the + main scheduler db). + + """ + from swh.scheduler.celery_backend.config import setup_log_handler + log_level = setup_log_handler( + loglevel=log_level, colorize=False, + format='[%(levelname)s] %(name)s -- %(message)s', + log_console=not no_stdout) + + ctx.ensure_object(dict) + + logger = logging.getLogger(__name__) + vault = None + conf = config.read(config_file, DEFAULT_CONFIG) + if 'vault' not in conf: + raise ValueError("missing 'vault' configuration") + + if database: + conf['vault']['cls'] = 'local' + conf['vault']['args']['db'] = database + elif url: + conf['vault']['cls'] = 'remote' + conf['vault']['args'] = {'url': url} + sched_conf = conf['vault'] + try: + logger.debug('Instanciating vault with %s' % ( + sched_conf)) + vault = get_vault(**conf) + except ValueError: + # it's the subcommand to decide whether not having a proper + # vault instance is a problem. + pass + + ctx.obj['vault'] = vault + ctx.obj['config'] = conf + ctx.obj['loglevel'] = log_level + + +@cli.command('api-server') +@click.option('--host', default='0.0.0.0', help="Host to run the server") +@click.option('--port', default=5005, type=click.INT, + help="Binding port of the server") +@click.option('--debug/--nodebug', default=True, + help="Indicates if the server should run in debug mode") +@click.pass_context +def serve(ctx, host, port, debug): + if ctx.obj['config']['vault']['cls'] == 'remote': + click.echo("The API server can only be started with a 'local' " + "configuration", err=True) + ctx.exit(1) + app = make_app(ctx.obj['config'], backend=ctx.obj['vault'], + debug=bool(debug)) + aiohttp.web.run_app(app, host=host, port=int(port)) + + +def main(): + return cli(auto_envvar_prefix='SWH_VAULT') + + +if __name__ == '__main__': + main()