diff --git a/requirements-swh.txt b/requirements-swh.txt index dcd0424..37ed732 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core[http] >= 0.0.60 +swh.core[http] >= 0.0.61 swh.model >= 0.0.27 diff --git a/setup.py b/setup.py index 32391bc..df591ba 100755 --- a/setup.py +++ b/setup.py @@ -1,68 +1,70 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2018 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from setuptools import setup, find_packages from os import path from io import open here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = 'requirements-%s.txt' % name else: reqf = 'requirements.txt' requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith('#'): continue requirements.append(line) return requirements setup( name='swh.objstorage', description='Software Heritage Object Storage', long_description=long_description, long_description_content_type='text/markdown', author='Software Heritage developers', author_email='swh-devel@inria.fr', url='https://forge.softwareheritage.org/diffusion/DOBJS', packages=find_packages(), install_requires=parse_requirements() + parse_requirements('swh'), setup_requires=['vcversioner'], extras_require={'testing': parse_requirements('test')}, vcversioner={}, include_package_data=True, entry_points=''' [console_scripts] swh-objstorage=swh.objstorage.cli:main + [swh.cli.subcommands] + objstorage=swh.objstorage.cli:cli ''', classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ 'Bug Reports': 'https://forge.softwareheritage.org/maniphest', 'Funding': 'https://www.softwareheritage.org/donate', 'Source': 'https://forge.softwareheritage.org/source/swh-objstorage', }, ) diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py index a3f5ad8..53810a2 100644 --- a/swh/objstorage/cli.py +++ b/swh/objstorage/cli.py @@ -1,83 +1,91 @@ # Copyright (C) 2015-2019 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import os import logging import time import click import aiohttp.web +from swh.core.cli import CONTEXT_SETTINGS + from swh.objstorage import get_objstorage from swh.objstorage.api.server import load_and_check_config, make_app -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) - -@click.group(context_settings=CONTEXT_SETTINGS) +@click.group(name='objstorage', context_settings=CONTEXT_SETTINGS) @click.option('--config-file', '-C', default=None, type=click.Path(exists=True, dir_okay=False,), help="Configuration file.") -@click.option('--log-level', '-l', default='INFO', - type=click.Choice(logging._nameToLevel.keys()), - help="Log level (default to INFO)") @click.pass_context -def cli(ctx, config_file, log_level): +def cli(ctx, config_file): + '''Software Heritage Objstorage tools. + ''' ctx.ensure_object(dict) - logging.basicConfig(level=log_level) cfg = load_and_check_config(config_file) ctx.obj['config'] = cfg - ctx.obj['log_level'] = log_level -@cli.command('serve') -@click.option('--host', default='0.0.0.0', help="Host to run the server") +@cli.command('rpc-serve') +@click.option('--host', default='0.0.0.0', + metavar='IP', show_default=True, + help="Host ip address to bind the server on") @click.option('--port', '-p', default=5003, type=click.INT, + metavar='PORT', show_default=True, help="Binding port of the server") @click.pass_context def serve(ctx, host, port): + '''Run a standalone objstorage server. + + This is not meant to be run on production systems. + ''' app = make_app(ctx.obj['config']) if ctx.obj['log_level'] == 'DEBUG': app.update(debug=True) aiohttp.web.run_app(app, host=host, port=int(port)) @cli.command('import') @click.argument('directory', required=True, nargs=-1) @click.pass_context def import_directories(ctx, directory): + '''Import a local directory in an existing objstorage. + ''' objstorage = get_objstorage(**ctx.obj['config']['objstorage']) nobj = 0 volume = 0 t0 = time.time() for dirname in directory: for root, _dirs, files in os.walk(dirname): for name in files: path = os.path.join(root, name) with open(path, 'rb') as f: objstorage.add(f.read()) volume += os.stat(path).st_size nobj += 1 click.echo('Imported %d files for a volume of %s bytes in %d seconds' % (nobj, volume, time.time()-t0)) @cli.command('fsck') @click.pass_context def fsck(ctx): + '''Check the objstorage is not corrupted. + ''' objstorage = get_objstorage(**ctx.obj['config']['objstorage']) for obj_id in objstorage: try: objstorage.check(obj_id) except objstorage.Error as err: logging.error(err) def main(): return cli(auto_envvar_prefix='SWH_OBJSTORAGE') if __name__ == '__main__': main()