diff --git a/docs/cli.rst b/docs/cli.rst index b48e535..2ba90a1 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -1,8 +1,8 @@ .. _swh-fuse-cli: Command-line interface ====================== -.. click:: swh.fuse.cli:cli +.. click:: swh.fuse.cli:fuse :prog: swh fuse :show-nested: diff --git a/setup.py b/setup.py index 764cc59..0ac1d89 100755 --- a/setup.py +++ b/setup.py @@ -1,71 +1,71 @@ #!/usr/bin/env python3 # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information from io import open from os import path from setuptools import find_packages, setup here = path.abspath(path.dirname(__file__)) # Get the long description from the README file with open(path.join(here, "README.rst"), encoding="utf-8") as f: long_description = f.read() def parse_requirements(name=None): if name: reqf = "requirements-%s.txt" % name else: reqf = "requirements.txt" requirements = [] if not path.exists(reqf): return requirements with open(reqf) as f: for line in f.readlines(): line = line.strip() if not line or line.startswith("#"): continue requirements.append(line) return requirements setup( name="swh.fuse", description="Software Heritage virtual file system", long_description=long_description, long_description_content_type="text/x-rst", python_requires=">=3.7", author="Software Heritage developers", author_email="swh-devel@inria.fr", url="https://forge.softwareheritage.org/source/swh-fuse", packages=find_packages(), # packages's modules install_requires=parse_requirements() + parse_requirements("swh"), tests_require=parse_requirements("test"), setup_requires=["setuptools-scm"], use_scm_version=True, extras_require={"testing": parse_requirements("test")}, include_package_data=True, entry_points=""" [swh.cli.subcommands] - fuse=swh.fuse.cli:cli + fuse=swh.fuse.cli """, classifiers=[ "Programming Language :: Python :: 3", "Intended Audience :: Developers", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: OS Independent", "Development Status :: 3 - Alpha", ], project_urls={ "Bug Reports": "https://forge.softwareheritage.org/maniphest", "Funding": "https://www.softwareheritage.org/donate", "Source": "https://forge.softwareheritage.org/source/swh-fuse", "Documentation": "https://docs.softwareheritage.org/devel/swh-fuse/", }, ) diff --git a/swh/fuse/cli.py b/swh/fuse/cli.py index 0d6ce38..336c9fd 100644 --- a/swh/fuse/cli.py +++ b/swh/fuse/cli.py @@ -1,120 +1,112 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import asyncio from contextlib import ExitStack import os # WARNING: do not import unnecessary things here to keep cli startup time under # control from pathlib import Path -from typing import Any, Dict, Tuple +from typing import Any, Dict import click from daemon import DaemonContext -# from swh.core import config +from swh.core import config from swh.core.cli import CONTEXT_SETTINGS +from swh.core.cli import swh as swh_cli_group from swh.model.cli import SWHIDParamType # All generic config code should reside in swh.core.config DEFAULT_CONFIG_PATH = os.environ.get( "SWH_CONFIG_FILE", os.path.join(click.get_app_dir("swh"), "global.yml") ) CACHE_HOME_DIR: Path = ( Path(os.environ["XDG_CACHE_HOME"]) if "XDG_CACHE_HOME" in os.environ else Path.home() / ".cache" ) -DEFAULT_CONFIG: Dict[str, Tuple[str, Any]] = { - "cache": ( - "dict", - { - "metadata": {"path": CACHE_HOME_DIR / "swh/fuse/metadata.sqlite"}, - "blob": {"path": CACHE_HOME_DIR / "swh/fuse/blob.sqlite"}, - }, - ), - "web-api": ( - "dict", - {"url": "https://archive.softwareheritage.org/api/1", "auth-token": None,}, - ), +DEFAULT_CONFIG: Dict[str, Any] = { + "cache": { + "metadata": {"path": CACHE_HOME_DIR / "swh/fuse/metadata.sqlite"}, + "blob": {"path": CACHE_HOME_DIR / "swh/fuse/blob.sqlite"}, + }, + "web-api": { + "url": "https://archive.softwareheritage.org/api/1", + "auth-token": None, + }, } -@click.group(name="fuse", context_settings=CONTEXT_SETTINGS) -# XXX conffile logic temporarily commented out due to: -# XXX https://forge.softwareheritage.org/T2632 -# @click.option( -# "-C", -# "--config-file", -# default=DEFAULT_CONFIG_PATH, -# type=click.Path(exists=True, dir_okay=False, path_type=str), -# help="YAML configuration file", -# ) +@swh_cli_group.group(name="fuse", context_settings=CONTEXT_SETTINGS) +@click.option( + "-C", + "--config-file", + default=None, + type=click.Path(exists=True, dir_okay=False, path_type=str), + help="YAML configuration file", +) @click.pass_context -def cli(ctx): +def fuse(ctx, config_file): """Software Heritage virtual file system""" - # # recursive merge not done by config.read - # conf = config.read_raw_config(config.config_basepath(config_file)) - # conf = config.merge_configs(DEFAULT_CONFIG, conf) - conf = {} + if not config_file and config.config_exists(DEFAULT_CONFIG_PATH): + config_file = DEFAULT_CONFIG_PATH + + if not config_file: + conf = DEFAULT_CONFIG + else: + # recursive merge not done by config.read + conf = config.read_raw_config(config.config_basepath(config_file)) + conf = config.merge_configs(DEFAULT_CONFIG, conf) ctx.ensure_object(dict) ctx.obj["config"] = conf -@cli.command() +@fuse.command(name="mount") @click.argument( "path", required=True, metavar="PATH", type=click.Path(exists=True, dir_okay=True, file_okay=False), ) @click.argument("swhids", nargs=-1, metavar="[SWHID]...", type=SWHIDParamType()) -@click.option( - "--config-file", - "-C", - default=None, - type=click.Path(exists=True, dir_okay=False,), - help="YAML configuration file", -) @click.option( "-f/-d", "--foreground/--daemon", default=False, help="whether to run FUSE attached to the console (foreground) " "or daemonized in the background (default: daemon)", ) @click.pass_context -def mount(ctx, swhids, path, config_file, foreground): +def mount(ctx, swhids, path, foreground): """Mount the Software Heritage archive at PATH If specified, objects referenced by the given SWHIDs will be prefetched and used to populate the virtual file system (VFS). Otherwise the VFS will be populated on-demand, when accessing its content. \b Example: \b $ mkdir swhfs $ swh fuse mount swhfs/ $ grep printf swhfs/archive/swh:1:cnt:c839dea9e8e6f0528b468214348fee8669b305b2 printf("Hello, World!"); $ """ - from swh.core import config from swh.fuse import fuse - conf = config.read(config_file, DEFAULT_CONFIG) with ExitStack() as stack: if not foreground: stack.enter_context(DaemonContext()) - asyncio.run(fuse.main(swhids, path, conf)) + asyncio.run(fuse.main(swhids, path, ctx.obj["config"])) diff --git a/swh/fuse/tests/conftest.py b/swh/fuse/tests/conftest.py index 4f0fdb9..b81f1f2 100644 --- a/swh/fuse/tests/conftest.py +++ b/swh/fuse/tests/conftest.py @@ -1,69 +1,76 @@ # Copyright (C) 2020 The Software Heritage developers # See the AUTHORS file at the top-level directory of this distribution # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information import json from multiprocessing import Process import os from pathlib import Path import subprocess from tempfile import NamedTemporaryFile, TemporaryDirectory import time from click.testing import CliRunner import pytest import yaml -from swh.fuse import cli +import swh.fuse.cli as cli from swh.fuse.tests.data.api_data import API_URL, MOCK_ARCHIVE @pytest.fixture def web_api_mock(requests_mock): for api_call, data in MOCK_ARCHIVE.items(): # Convert Python dict JSON into a string (only for non-raw API call) if not api_call.endswith("raw/"): data = json.dumps(data) requests_mock.get(f"{API_URL}/{api_call}", text=data) return requests_mock @pytest.fixture def fuse_mntdir(web_api_mock): tmpdir = TemporaryDirectory(suffix=".swh-fuse-test") tmpfile = NamedTemporaryFile(suffix=".swh-fuse-test.yml") config = { "cache": {"metadata": {"in-memory": True}, "blob": {"in-memory": True}}, "web-api": {"url": API_URL, "auth-token": None}, } # Run FUSE in foreground mode but in a separate process, so it does not # block execution and remains easy to kill during teardown def fuse_process(tmpdir, tmpfile): with tmpdir as mntdir, tmpfile as config_path: config_path = Path(config_path.name) config_path.write_text(yaml.dump(config)) CliRunner().invoke( - cli.mount, args=[mntdir, "--foreground", "--config-file", config_path,], + cli.fuse, + args=[ + "--config-file", + str(config_path), + "mount", + mntdir, + "--foreground", + ], ) fuse = Process(target=fuse_process, args=[tmpdir, tmpfile]) fuse.start() # Wait max 3 seconds for the FUSE to correctly mount for i in range(30): try: root = os.listdir(tmpdir.name) if root: break except FileNotFoundError: pass time.sleep(0.1) else: raise FileNotFoundError(f"Could not mount FUSE in {tmpdir.name}") yield Path(tmpdir.name) subprocess.run(["fusermount", "-u", tmpdir.name], check=True) fuse.join()