diff --git a/swh/vault/__init__.py b/swh/vault/__init__.py --- a/swh/vault/__init__.py +++ b/swh/vault/__init__.py @@ -16,6 +16,7 @@ BACKEND_TYPES: Dict[str, str] = { "remote": ".api.client.RemoteVaultClient", "local": ".backend.VaultBackend", + "memory": ".in_memory_backend.InMemoryVaultBackend", } diff --git a/swh/vault/backend.py b/swh/vault/backend.py --- a/swh/vault/backend.py +++ b/swh/vault/backend.py @@ -370,7 +370,7 @@ @db_transaction() def fetch( self, obj_type: str, obj_id: ObjectId, raise_notfound=True, db=None, cur=None - ): + ) -> Optional[bytes]: """Retrieve a bundle from the cache""" hex_id, obj_id = self._compute_ids(obj_id) available = self.is_available(obj_type, obj_id, cur=cur) diff --git a/swh/vault/cli.py b/swh/vault/cli.py --- a/swh/vault/cli.py +++ b/swh/vault/cli.py @@ -1,28 +1,116 @@ -# Copyright (C) 2015-2020 The Software Heritage developers +# Copyright (C) 2015-2021 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from __future__ import annotations + # WARNING: do not import unnecessary things here to keep cli startup time under # control import logging +from typing import TYPE_CHECKING, Optional import click from swh.core.cli import CONTEXT_SETTINGS, AliasedGroup from swh.core.cli import swh as swh_cli_group -CFG_HELP = """Software Heritage Vault RPC server.""" +if TYPE_CHECKING: + import io + + from swh.model.identifiers import CoreSWHID + + +class SwhidParamType(click.ParamType): + name = "swhid" + + def convert(self, value, param, ctx): + from swh.model.exceptions import ValidationError + from swh.model.identifiers import CoreSWHID + + try: + return CoreSWHID.from_string(value) + except ValidationError: + self.fail(f"expected core SWHID, got {value!r}", param, ctx) @swh_cli_group.group(name="vault", context_settings=CONTEXT_SETTINGS, cls=AliasedGroup) @click.pass_context def vault(ctx): """Software Heritage Vault tools.""" - pass -@vault.command(name="rpc-serve", help=CFG_HELP) +@vault.command() +@click.option( + "--config-file", + "-C", + default=None, + metavar="CONFIGFILE", + type=click.Path(exists=True, dir_okay=False,), + help="Configuration file.", +) +@click.argument("swhid", type=SwhidParamType()) +@click.argument("outfile", type=click.File("wb")) +@click.option( + "--cooker-type", + type=click.Choice(["flat", "gitfast"]), + help="Selects which cooker to use, when there is more than one available " + "for the given object type.", +) +@click.pass_context +def cook( + ctx, + config_file: str, + swhid: CoreSWHID, + outfile: io.RawIOBase, + cooker_type: Optional[str], +): + """ + Runs a vault cooker for a single object (identified by a SWHID), + and outputs it to the given file. + """ + from swh.core import config + from swh.storage import get_storage + + from .cookers import COOKER_TYPES, get_cooker_cls + from .in_memory_backend import InMemoryVaultBackend + + conf = config.read(config_file) + + supported_object_types = {name.split("_")[0] for name in COOKER_TYPES} + if swhid.object_type.name.lower() not in supported_object_types: + raise click.ClickException( + f"No cooker available for {swhid.object_type.name} objects." + ) + + cooker_name = swhid.object_type.name.lower() + + if cooker_type: + cooker_name = f"{cooker_name}_{cooker_type}" + if cooker_name not in COOKER_TYPES: + raise click.ClickException( + f"{swhid.object_type.name.lower()} objects do not have " + f"a {cooker_type} cooker." + ) + else: + if cooker_name not in COOKER_TYPES: + raise click.ClickException( + f"{swhid.object_type.name.lower()} objects need " + f"an explicit --cooker-type." + ) + + backend = InMemoryVaultBackend() + storage = get_storage(**conf["storage"]) + cooker_cls = get_cooker_cls(cooker_name) + cooker = cooker_cls(cooker_name, swhid.object_id, backend, storage) + cooker.cook() + + bundle = backend.fetch(cooker_name, swhid.object_id) + assert bundle, "Cooker did not write a bundle to the backend." + outfile.write(bundle) + + +@vault.command(name="rpc-serve") @click.option( "--config-file", "-C", @@ -52,6 +140,7 @@ ) @click.pass_context def serve(ctx, config_file, host, port, debug): + """Software Heritage Vault RPC server.""" import aiohttp from swh.vault.api.server import make_app_from_configfile diff --git a/swh/vault/in_memory_backend.py b/swh/vault/in_memory_backend.py new file mode 100644 --- /dev/null +++ b/swh/vault/in_memory_backend.py @@ -0,0 +1,53 @@ +# Copyright (C) 2017-2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Any, Dict, List, Optional, Tuple, Union + +from swh.model.hashutil import hash_to_bytes + +from .cache import VaultCache + +ObjectId = Union[str, bytes] + + +class InMemoryVaultBackend: + """Stub vault backend, for use in the CLI.""" + + def __init__(self): + self._cache = VaultCache(cls="memory") + + def fetch(self, obj_type: str, obj_id: ObjectId) -> Optional[bytes]: + return self._cache.get(obj_type, obj_id) + + def cook( + self, obj_type: str, obj_id: ObjectId, email: Optional[str] = None + ) -> Dict[str, Any]: + raise NotImplementedError("InMemoryVaultBackend.cook()") + + def progress(self, obj_type: str, obj_id: ObjectId): + raise NotImplementedError("InMemoryVaultBackend.progress()") + + # Cookers endpoints + + def set_progress(self, obj_type: str, obj_id: ObjectId, progress: str) -> None: + pass + + def set_status(self, obj_type: str, obj_id: ObjectId, status: str) -> None: + pass + + def put_bundle(self, obj_type: str, obj_id: ObjectId, bundle) -> bool: + self._cache.add(obj_type, hash_to_bytes(obj_id), bundle) + return True + + def send_notif(self, obj_type: str, obj_id: ObjectId): + pass + + # Batch endpoints + + def batch_cook(self, batch: List[Tuple[str, str]]) -> int: + raise NotImplementedError("InMemoryVaultBackend.batch_cook()") + + def batch_progress(self, batch_id: int) -> Dict[str, Any]: + pass diff --git a/swh/vault/interface.py b/swh/vault/interface.py --- a/swh/vault/interface.py +++ b/swh/vault/interface.py @@ -19,7 +19,7 @@ """ @remote_api_endpoint("fetch") - def fetch(self, obj_type: str, obj_id: ObjectId) -> Dict[str, Any]: + def fetch(self, obj_type: str, obj_id: ObjectId) -> Optional[bytes]: """Fetch information from a bundle""" ... @@ -43,7 +43,7 @@ ... @remote_api_endpoint("set_status") - def set_status(self, obj_type: str, obj_id: ObjectId, status: str) -> None: + def set_status(self, obj_type: str, obj_id: ObjectId, status: str) -> bool: """Set the cooking status of a bundle""" ... diff --git a/swh/vault/tests/test_cli.py b/swh/vault/tests/test_cli.py new file mode 100644 --- /dev/null +++ b/swh/vault/tests/test_cli.py @@ -0,0 +1,102 @@ +# Copyright (C) 2021 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import tempfile +from unittest.mock import MagicMock + +import click +import click.testing +import pytest + +from swh.vault.cli import vault as vault_cli_group +from swh.vault.cookers.base import BaseVaultCooker +from swh.vault.in_memory_backend import InMemoryVaultBackend + + +def test_cook_unsupported_swhid(): + runner = click.testing.CliRunner() + + result = runner.invoke(vault_cli_group, ["cook", "swh:1:dir:f00b4r", "-"]) + assert isinstance(result.exception, SystemExit) + assert "expected core SWHID" in result.stdout + + result = runner.invoke(vault_cli_group, ["cook", "swh:1:ori:" + "0" * 40, "-"]) + assert isinstance(result.exception, SystemExit) + assert "expected core SWHID" in result.stdout + + result = runner.invoke(vault_cli_group, ["cook", "swh:1:cnt:" + "0" * 40, "-"]) + assert isinstance(result.exception, SystemExit) + assert "No cooker available for CONTENT" in result.stdout + + +def test_cook_unknown_cooker(): + runner = click.testing.CliRunner() + + result = runner.invoke( + vault_cli_group, + ["cook", "swh:1:dir:" + "0" * 40, "-", "--cooker-type", "gitfast"], + ) + assert isinstance(result.exception, SystemExit) + assert "do not have a gitfast cooker" in result.stdout + + result = runner.invoke(vault_cli_group, ["cook", "swh:1:rev:" + "0" * 40, "-"]) + assert isinstance(result.exception, SystemExit) + assert "explicit --cooker-type" in result.stdout + + +@pytest.mark.parametrize( + "obj_type,cooker_name_suffix,swhid_type", + [("directory", "", "dir"), ("revision", "gitfast", "rev"),], +) +def test_cook_directory(obj_type, cooker_name_suffix, swhid_type, mocker): + storage = object() + mocker.patch("swh.storage.get_storage", return_value=storage) + + backend = MagicMock(spec=InMemoryVaultBackend) + backend.fetch.return_value = b"bundle content" + mocker.patch( + "swh.vault.in_memory_backend.InMemoryVaultBackend", return_value=backend + ) + + cooker = MagicMock(spec=BaseVaultCooker) + cooker_cls = MagicMock(return_value=cooker) + mocker.patch("swh.vault.cookers.get_cooker_cls", return_value=cooker_cls) + + runner = click.testing.CliRunner() + + with tempfile.NamedTemporaryFile("a", suffix=".yml") as config_fd: + config_fd.write('{"storage": {}}') + config_fd.seek(0) + if cooker_name_suffix: + result = runner.invoke( + vault_cli_group, + [ + "cook", + f"swh:1:{swhid_type}:{'0'*40}", + "-", + "-C", + config_fd.name, + "--cooker-type", + cooker_name_suffix, + ], + ) + else: + result = runner.invoke( + vault_cli_group, + ["cook", f"swh:1:{swhid_type}:{'0'*40}", "-", "-C", config_fd.name], + ) + + if result.exception is not None: + raise result.exception + + cooker_cls.assert_called_once_with( + f"{obj_type}_{cooker_name_suffix}" if cooker_name_suffix else obj_type, + b"\x00" * 20, + backend, + storage, + ) + cooker.cook.assert_called_once_with() + + assert result.stdout_bytes == b"bundle content"