diff --git a/requirements-swh.txt b/requirements-swh.txt index e6688a3..544f1ac 100644 --- a/requirements-swh.txt +++ b/requirements-swh.txt @@ -1,2 +1,2 @@ -swh.core[http] >= 0.0.65 +swh.core[http] >= 0.3 swh.model >= 0.0.27 diff --git a/setup.py b/setup.py index ad4d12c..91c4fd0 100755 --- a/setup.py +++ b/setup.py @@ -1,72 +1,70 @@ #!/usr/bin/env python3 # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.md"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.objstorage", description="Software Heritage Object Storage", long_description=long_description, long_description_content_type="text/markdown", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/diffusion/DOBJS", packages=find_packages(), install_requires=parse_requirements() + parse_requirements("swh"), setup_requires=["setuptools-scm"], use_scm_version=True, extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" - [console_scripts] - swh-objstorage=swh.objstorage.cli:main [swh.cli.subcommands] - objstorage=swh.objstorage.cli:cli + objstorage=swh.objstorage.cli """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 5 - Production/Stable", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-objstorage", "Documentation": "https://docs.softwareheritage.org/devel/swh-objstorage/", }, ) diff --git a/swh/objstorage/cli.py b/swh/objstorage/cli.py index 9fc8670..3180b88 100644 --- a/swh/objstorage/cli.py +++ b/swh/objstorage/cli.py @@ -1,126 +1,130 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import logging # WARNING: do not import unnecessary things here to keep cli startup time under # control import os import time import click -from swh.core.cli import CONTEXT_SETTINGS +from swh.core.cli import CONTEXT_SETTINGS, swh as swh_cli_group -@click.group(name="objstorage", context_settings=CONTEXT_SETTINGS) +@swh_cli_group.group(name="objstorage", context_settings=CONTEXT_SETTINGS) @click.option( "--config-file", "-C", default=None, type=click.Path(exists=True, dir_okay=False,), help="Configuration file.", ) @click.pass_context -def cli(ctx, config_file): +def objstorage_cli_group(ctx, config_file): """Software Heritage Objstorage tools. """ from swh.core import config if not config_file: config_file = os.environ.get("SWH_CONFIG_FILENAME") if config_file: if not os.path.exists(config_file): raise ValueError("%s does not exist" % config_file) conf = config.read(config_file) else: conf = {} ctx.ensure_object(dict) ctx.obj["config"] = conf -@cli.command("rpc-serve") +# for BW compat +cli = objstorage_cli_group + + +@objstorage_cli_group.command("rpc-serve") @click.option( "--host", default="0.0.0.0", metavar="IP", show_default=True, help="Host ip address to bind the server on", ) @click.option( "--port", "-p", default=5003, type=click.INT, metavar="PORT", show_default=True, help="Binding port of the server", ) @click.pass_context def serve(ctx, host, port): """Run a standalone objstorage server. This is not meant to be run on production systems. """ import aiohttp.web from swh.objstorage.api.server import make_app, validate_config app = make_app(validate_config(ctx.obj["config"])) if ctx.obj["log_level"] == "DEBUG": app.update(debug=True) aiohttp.web.run_app(app, host=host, port=int(port)) -@cli.command("import") +@objstorage_cli_group.command("import") @click.argument("directory", required=True, nargs=-1) @click.pass_context def import_directories(ctx, directory): """Import a local directory in an existing objstorage. """ from swh.objstorage.factory import get_objstorage objstorage = get_objstorage(**ctx.obj["config"]["objstorage"]) nobj = 0 volume = 0 t0 = time.time() for dirname in directory: for root, _dirs, files in os.walk(dirname): for name in files: path = os.path.join(root, name) with open(path, "rb") as f: objstorage.add(f.read()) volume += os.stat(path).st_size nobj += 1 click.echo( "Imported %d files for a volume of %s bytes in %d seconds" % (nobj, volume, time.time() - t0) ) -@cli.command("fsck") +@objstorage_cli_group.command("fsck") @click.pass_context def fsck(ctx): """Check the objstorage is not corrupted. """ from swh.objstorage.factory import get_objstorage objstorage = get_objstorage(**ctx.obj["config"]["objstorage"]) for obj_id in objstorage: try: objstorage.check(obj_id) except objstorage.Error as err: logging.error(err) def main(): return cli(auto_envvar_prefix="SWH_OBJSTORAGE") if __name__ == "__main__": main()