diff --git a/PKG-INFO b/PKG-INFO index bc3f006..79689b9 100644 --- a/PKG-INFO +++ b/PKG-INFO @@ -1,31 +1,31 @@ Metadata-Version: 2.1 Name: swh.vault -Version: 0.4.0 +Version: 0.5.0 Summary: Software Heritage vault Home-page: https://forge.softwareheritage.org/diffusion/DVAU/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-vault Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-vault/ Description: swh-vault ========= User-facing service that allows to retrieve parts of the archive as self-contained bundles. See the [documentation](https://docs.softwareheritage.org/devel/swh-vault/index.html) for more details. Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/debian/changelog b/debian/changelog index bfe06ac..863c356 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,371 +1,376 @@ -swh-vault (0.4.0-1~swh1~bpo10+1) buster-swh; urgency=medium +swh-vault (0.5.0-1~swh1) unstable-swh; urgency=medium - * Rebuild for buster-swh + * New upstream release 0.5.0 - (tagged by Antoine R. Dumont + (@ardumont) on 2020-12-08 15:58:26 + +0100) + * Upstream changes: - v0.5.0 - vault: Remove deprecated + services default config - cli: Remove deprecated logging + configuration - -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 12:53:53 +0000 + -- Software Heritage autobuilder (on jenkins-debian1) Tue, 08 Dec 2020 15:01:11 +0000 swh-vault (0.4.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.4.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-23 13:50:22 +0100) * Upstream changes: - v0.4.0 - requirements-test.txt: Drop no longer needed test dependency - swh.vault.tests.conftest: Drop dead code -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 12:52:25 +0000 swh-vault (0.3.4-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.4 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-23 11:35:47 +0100) * Upstream changes: - v0.3.4 - test_server: Fix exception structure - conftest: Explicitely declare aiohttp pytest plugin use -- Software Heritage autobuilder (on jenkins-debian1) Mon, 23 Nov 2020 10:37:50 +0000 swh-vault (0.3.3-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.3 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-18 18:02:35 +0100) * Upstream changes: - v0.3.3 - Fix api.server configuration adaptation issue -- Software Heritage autobuilder (on jenkins-debian1) Wed, 18 Nov 2020 18:40:45 +0000 swh-vault (0.3.1-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.1 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-17 17:46:37 +0100) * Upstream changes: - v0.3.1 - test_server: Simplify test server initialization to the minimum -- Software Heritage autobuilder (on jenkins-debian1) Tue, 17 Nov 2020 16:54:22 +0000 swh-vault (0.3.0-1~swh2) unstable-swh; urgency=medium * Fix dependency release -- Antoine R. Dumont (@ardumont) Tue, 17 Nov 2020 16:54:03 +0000 swh-vault (0.3.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.3.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-11-13 12:10:09 +0100) * Upstream changes: - v0.3.0 - Refactor vault configuration without the args indirection - vault.server: Introduce typed VaultInterface - Replace file modes literals to DentryPerms enum - Add tests on current configuration check for cooker instantiation - api.server: Add types and tests on configuration checks - swh.vault: Unify get_vault factory function with other factories - vault.tests: Make postgresql fixture faster -- Software Heritage autobuilder (on jenkins-debian1) Tue, 17 Nov 2020 16:22:52 +0000 swh-vault (0.2.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.2.0 - (tagged by Antoine R. Dumont (@ardumont) on 2020-10-19 09:52:04 +0200) * Upstream changes: - v0.2.0 - vault.config: Adapt scheduler configuration structure - test_cookers: Turn git_loader into a pytest fixture - tests: Fix loader git instantiation - tox.ini: pin black to the pre-commit version (19.10b0) to avoid flip- flops - Run isort after the CLI import changes -- Software Heritage autobuilder (on jenkins-debian1) Mon, 19 Oct 2020 07:54:03 +0000 swh-vault (0.1.0-1~swh1) unstable-swh; urgency=medium * New upstream release 0.1.0 - (tagged by David Douard on 2020-09-25 12:34:43 +0200) * Upstream changes: - v0.1.0 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 25 Sep 2020 10:37:22 +0000 swh-vault (0.0.35-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.35 - (tagged by David Douard on 2020-09-11 15:15:26 +0200) * Upstream changes: - v0.0.35 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 11 Sep 2020 13:18:50 +0000 swh-vault (0.0.34-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.34 - (tagged by Antoine Lambert on 2020-08-18 13:55:51 +0200) * Upstream changes: - version 0.0.34 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 18 Aug 2020 11:58:22 +0000 swh-vault (0.0.33-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.33 - (tagged by Valentin Lorentz on 2020-05-05 17:56:33 +0200) * Upstream changes: - v0.0.33 - * Use swh-storage validation proxy. - * Use model objects to send to storage - * Add a pyproject.toml file to target py37 for black - * setup: Update the minimum required runtime python3 version - * setup.py: add documentation link - * Raise NotFoundExc within our RPC framework instead of returning 404. -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 May 2020 15:59:51 +0000 swh-vault (0.0.32-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.32 - (tagged by Antoine Lambert on 2020-02-05 13:00:19 +0100) * Upstream changes: - version 0.0.32 -- Software Heritage autobuilder (on jenkins-debian1) Wed, 05 Feb 2020 12:16:16 +0000 swh-vault (0.0.31-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.31 - (tagged by Stefano Zacchiroli on 2019-11-05 17:24:43 +0100) * Upstream changes: - v0.0.31 - * typing: minimal changes to make a no-op mypy run pass - * Remove indirection swh.vault.api.wsgi - * tox.ini: Fix py3 environment to use packaged tests - * CLI: drop obsolete alias "serve" for "rpc- serve" -- Software Heritage autobuilder (on jenkins-debian1) Tue, 05 Nov 2019 16:44:29 +0000 swh-vault (0.0.30-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.30 - (tagged by Antoine Lambert on 2019-07-29 11:17:23 +0200) * Upstream changes: - version 0.0.30 -- Software Heritage autobuilder (on jenkins-debian1) Mon, 29 Jul 2019 09:22:02 +0000 swh-vault (0.0.29-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.29 - (tagged by Antoine Lambert on 2019-05-23 11:39:12 +0200) * Upstream changes: - version 0.0.29 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 May 2019 09:46:57 +0000 swh-vault (0.0.28-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.28 - (tagged by Antoine Lambert on 2019-05-23 11:00:51 +0200) * Upstream changes: - version 0.0.28 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 23 May 2019 09:05:34 +0000 swh-vault (0.0.27-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.27 - (tagged by Antoine Lambert on 2019-05-07 14:44:26 +0200) * Upstream changes: - version 0.0.27 -- Software Heritage autobuilder (on jenkins-debian1) Tue, 07 May 2019 12:54:35 +0000 swh-vault (0.0.26-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.26 - (tagged by Antoine Lambert on 2019-04-26 11:59:23 +0200) * Upstream changes: - version 0.0.26 -- Software Heritage autobuilder (on jenkins-debian1) Fri, 26 Apr 2019 10:06:45 +0000 swh-vault (0.0.25-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.25 - (tagged by Antoine R. Dumont (@ardumont) on 2019-03-29 12:19:19 +0100) * Upstream changes: - v0.0.25 - master vault.backend: Migrate email address to bot@swh.org - API: use default's APIError exception instead of the VaultAPIError - Remove debian packaging from master branch -- Software Heritage autobuilder (on jenkins-debian1) Fri, 29 Mar 2019 11:28:28 +0000 swh-vault (0.0.24-1~swh3) unstable-swh; urgency=low * d/control: Update missing build dependency on postgresql-contrib -- Antoine Romain Dumont Mon, 18 Feb 2019 16:20:50 +0100 swh-vault (0.0.24-1~swh2) unstable-swh; urgency=low * d/control: Update missing build dependency on git * d/rules: Sanitize build locale -- Antoine Romain Dumont Mon, 18 Feb 2019 16:04:50 +0100 swh-vault (0.0.24-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.24 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-18 15:21:31 +0100) * Upstream changes: - v0.0.24 - MANIFEST.in: Fix packaging to include the sql schema definitions -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Feb 2019 14:25:33 +0000 swh-vault (0.0.23-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.23 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-18 14:39:25 +0100) * Upstream changes: - v0.0.23 - test_cookers: Fix commit behavior when committing to another branch - Rewrite tests using pytest's fixtures and adapt them to recent refactorings - Normalize the configuration of VaultBackend and cooker - Make it possible to specify the config file via SWH_CONFIG_FILENAME env var - Refactor the VaultBackend to use BaseDb and pool-based db access - Add a swh.vault.api.wsgi module to instanciate the (singleton) wsgi app object -- Software Heritage autobuilder (on jenkins-debian1) Mon, 18 Feb 2019 13:48:28 +0000 swh-vault (0.0.22-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.22 - (tagged by Antoine R. Dumont (@ardumont) on 2019-02-14 10:12:41 +0100) * Upstream changes: - v0.0.22 - api/server: Do not read configuration at each request -- Software Heritage autobuilder (on jenkins-debian1) Thu, 14 Feb 2019 09:16:23 +0000 swh-vault (0.0.21-1~swh1) unstable-swh; urgency=medium * New upstream release 0.0.21 - (tagged by David Douard on 2019-02-07 17:38:49 +0100) * Upstream changes: - v0.0.21 -- Software Heritage autobuilder (on jenkins-debian1) Thu, 07 Feb 2019 16:44:51 +0000 swh-vault (0.0.20-1~swh1) unstable-swh; urgency=medium * v0.0.20 * swh.vault: Open a get_vault instantiation function * swh.vault.api.client: Permit to specify the query timeout option * swh.storage doesn't expose a db attribute any longer -- Antoine R. Dumont (@ardumont) Thu, 24 May 2018 12:31:50 +0200 swh-vault (0.0.19-1~swh1) unstable-swh; urgency=medium * version 0.0.19 -- Antoine Pietri Thu, 03 May 2018 17:49:18 +0200 swh-vault (0.0.18-1~swh1) unstable-swh; urgency=medium * version 0.0.18 -- Antoine Pietri Thu, 03 May 2018 17:10:24 +0200 swh-vault (0.0.17-1~swh1) unstable-swh; urgency=medium * version 0.0.17 -- Antoine Pietri Thu, 03 May 2018 13:16:59 +0200 swh-vault (0.0.16-1~swh1) unstable-swh; urgency=medium * version 0.0.16 -- Antoine Pietri Wed, 02 May 2018 13:41:05 +0200 swh-vault (0.0.15-1~swh1) unstable-swh; urgency=medium * version 0.0.15 -- Antoine Pietri Fri, 27 Apr 2018 18:46:06 +0200 swh-vault (0.0.14-1~swh1) unstable-swh; urgency=medium * version 0.0.14 -- Antoine Pietri Fri, 27 Apr 2018 17:11:50 +0200 swh-vault (0.0.13-1~swh1) unstable-swh; urgency=medium * version 0.0.13 -- Antoine Pietri Wed, 25 Apr 2018 15:52:33 +0200 swh-vault (0.0.12-1~swh1) unstable-swh; urgency=medium * version 0.0.12 -- Antoine Pietri Wed, 21 Feb 2018 15:30:25 +0100 swh-vault (0.0.11-1~swh1) unstable-swh; urgency=medium * version 0.0.11 -- Antoine Pietri Fri, 16 Feb 2018 16:09:10 +0100 swh-vault (0.0.10-1~swh1) unstable-swh; urgency=medium * version 0.0.10 -- Antoine Pietri Thu, 15 Feb 2018 16:08:05 +0100 swh-vault (0.0.9-1~swh1) unstable-swh; urgency=medium * version 0.0.9 -- Antoine Pietri Thu, 01 Feb 2018 18:21:29 +0100 swh-vault (0.0.8-1~swh1) unstable-swh; urgency=medium * version 0.0.8 -- Antoine Pietri Wed, 31 Jan 2018 17:54:55 +0100 swh-vault (0.0.7-1~swh1) unstable-swh; urgency=medium * version 0.0.7 -- Antoine Pietri Tue, 30 Jan 2018 18:21:07 +0100 swh-vault (0.0.6-1~swh1) unstable-swh; urgency=medium * version 0.0.6 -- Antoine Pietri Tue, 09 Jan 2018 16:37:41 +0100 swh-vault (0.0.5-1~swh1) unstable-swh; urgency=medium * version 0.0.5 -- Antoine Pietri Thu, 14 Dec 2017 19:33:01 +0100 swh-vault (0.0.4-1~swh1) unstable-swh; urgency=medium * version 0.0.4 -- Antoine Pietri Fri, 08 Dec 2017 15:33:54 +0100 swh-vault (0.0.3-1~swh1) unstable-swh; urgency=medium * version 0.0.3 -- Antoine Pietri Fri, 01 Dec 2017 15:31:34 +0100 swh-vault (0.0.2-1~swh1) unstable-swh; urgency=medium * version 0.0.2 -- Antoine Pietri Thu, 30 Nov 2017 15:50:43 +0100 swh-vault (0.0.1-1~swh1) unstable-swh; urgency=medium * Initial release * version 0.0.1 -- Antoine Pietri Mon, 13 Nov 2017 16:22:47 +0100 diff --git a/swh.vault.egg-info/PKG-INFO b/swh.vault.egg-info/PKG-INFO index bc3f006..79689b9 100644 --- a/swh.vault.egg-info/PKG-INFO +++ b/swh.vault.egg-info/PKG-INFO @@ -1,31 +1,31 @@ Metadata-Version: 2.1 Name: swh.vault -Version: 0.4.0 +Version: 0.5.0 Summary: Software Heritage vault Home-page: https://forge.softwareheritage.org/diffusion/DVAU/ Author: Software Heritage developers Author-email: swh-devel@inria.fr License: UNKNOWN Project-URL: Bug Reports, https://forge.softwareheritage.org/maniphest Project-URL: Funding, https://www.softwareheritage.org/donate Project-URL: Source, https://forge.softwareheritage.org/source/swh-vault Project-URL: Documentation, https://docs.softwareheritage.org/devel/swh-vault/ Description: swh-vault ========= User-facing service that allows to retrieve parts of the archive as self-contained bundles. See the [documentation](https://docs.softwareheritage.org/devel/swh-vault/index.html) for more details. Platform: UNKNOWN Classifier: Programming Language :: Python :: 3 Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3) Classifier: Operating System :: OS Independent Classifier: Development Status :: 5 - Production/Stable Requires-Python: >=3.7 Description-Content-Type: text/markdown Provides-Extra: testing diff --git a/swh/vault/api/server.py b/swh/vault/api/server.py index cbe0329..baa565f 100644 --- a/swh/vault/api/server.py +++ b/swh/vault/api/server.py @@ -1,135 +1,128 @@ # Copyright (C) 2016-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from __future__ import annotations import asyncio import os from typing import Any, Dict, Optional import aiohttp.web from swh.core.api.asynchronous import RPCServerApp from swh.core.config import config_basepath, merge_configs, read_raw_config from swh.vault import get_vault as get_swhvault from swh.vault.backend import NotFoundExc from swh.vault.interface import VaultInterface +# do not define default services here DEFAULT_CONFIG = { - "storage": {"cls": "remote", "url": "http://localhost:5002/"}, - "cache": { - "cls": "pathslicing", - "root": "/srv/softwareheritage/vault", - "slicing": "0:1/1:5", - }, "client_max_size": 1024 ** 3, - "vault": {"cls": "local", "db": "dbname=softwareheritage-vault-dev",}, - "scheduler": {"cls": "remote", "url": "http://localhost:5008/"}, } vault = None app = None def get_vault(config: Optional[Dict[str, Any]] = None) -> VaultInterface: global vault if not vault: assert config is not None vault = get_swhvault(**config) return vault class VaultServerApp(RPCServerApp): client_exception_classes = (NotFoundExc,) @asyncio.coroutine def index(request): return aiohttp.web.Response(body="SWH Vault API server") def check_config(cfg: Dict[str, Any]) -> Dict[str, Any]: """Ensure the configuration is ok to run a local vault server, and propagate defaults. Raises: EnvironmentError if the configuration is not for local instance ValueError if one of the following keys is missing: vault, cache, storage, scheduler Returns: New configuration dict to instantiate a local vault server instance. """ cfg = cfg.copy() if "vault" not in cfg: raise ValueError("missing 'vault' configuration") vcfg = cfg["vault"] if vcfg["cls"] != "local": raise EnvironmentError( "The vault backend can only be started with a 'local' configuration", ) # TODO: Soft-deprecation of args key. Remove when ready. vcfg.update(vcfg.get("args", {})) # Default to top-level value if any if "cache" not in vcfg: vcfg["cache"] = cfg.get("cache") if "storage" not in vcfg: vcfg["storage"] = cfg.get("storage") if "scheduler" not in vcfg: vcfg["scheduler"] = cfg.get("scheduler") if "client_max_size" not in vcfg: vcfg["client_max_size"] = cfg.get("client_max_size") for key in ("cache", "storage", "scheduler"): if not vcfg.get(key): raise ValueError(f"invalid configuration: missing {key} config entry.") return vcfg def make_app(config: Dict[str, Any]) -> VaultServerApp: """Ensure the configuration is ok, then instantiate the server application """ config = check_config(config) app = VaultServerApp( __name__, backend_class=VaultInterface, backend_factory=lambda: get_vault(config), client_max_size=config["client_max_size"], ) app.router.add_route("GET", "/", index) return app def make_app_from_configfile( config_path: Optional[str] = None, **kwargs ) -> VaultServerApp: """Load and check configuration if ok, then instantiate (once) a vault server application. """ global app if not app: config_path = os.environ.get("SWH_CONFIG_FILENAME", config_path) if not config_path: raise ValueError("Missing configuration path.") if not os.path.isfile(config_path): raise ValueError(f"Configuration path {config_path} should exist.") app_config = read_raw_config(config_basepath(config_path)) app_config = merge_configs(DEFAULT_CONFIG, app_config) app = make_app(app_config) return app if __name__ == "__main__": print("Deprecated. Use swh-vault ") diff --git a/swh/vault/cli.py b/swh/vault/cli.py index f3e1279..e99322b 100644 --- a/swh/vault/cli.py +++ b/swh/vault/cli.py @@ -1,86 +1,76 @@ # Copyright (C) 2015-2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information # WARNING: do not import unnecessary things here to keep cli startup time under # control import logging import click from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.core.cli import swh as swh_cli_group CFG_HELP = """Software Heritage Vault RPC server.""" @swh_cli_group.group(name="vault", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.pass_context def vault(ctx): """Software Heritage Vault tools.""" pass @vault.command(name="rpc-serve", help=CFG_HELP) @click.option( "--config-file", "-C", default=None, metavar="CONFIGFILE", type=click.Path(exists=True, dir_okay=False,), help="Configuration file.", ) -@click.option( - "--no-stdout", is_flag=True, default=False, help="Do NOT output logs on the console" -) @click.option( "--host", default="0.0.0.0", metavar="IP", show_default=True, help="Host ip address to bind the server on", ) @click.option( "--port", default=5005, type=click.INT, metavar="PORT", help="Binding port of the server", ) @click.option( "--debug/--no-debug", default=True, help="Indicates if the server should run in debug mode", ) @click.pass_context -def serve(ctx, config_file, no_stdout, host, port, debug): +def serve(ctx, config_file, host, port, debug): import aiohttp - from swh.scheduler.celery_backend.config import setup_log_handler from swh.vault.api.server import make_app_from_configfile ctx.ensure_object(dict) - setup_log_handler( - loglevel=ctx.obj.get("log_level", logging.INFO), - colorize=False, - format="[%(levelname)s] %(name)s -- %(message)s", - log_console=not no_stdout, - ) try: app = make_app_from_configfile(config_file, debug=debug) except EnvironmentError as e: click.echo(e.msg, err=True) ctx.exit(1) aiohttp.web.run_app(app, host=host, port=int(port)) def main(): logging.basicConfig() return serve(auto_envvar_prefix="SWH_VAULT") if __name__ == "__main__": main()