diff --git a/mypy.ini b/mypy.ini --- a/mypy.ini +++ b/mypy.ini @@ -5,11 +5,17 @@ # 3rd party libraries without stubs (yet) +[mypy-daemon.*] +ignore_missing_imports = True + [mypy-pkg_resources.*] ignore_missing_imports = True [mypy-pytest.*] ignore_missing_imports = True -# [mypy-add_your_lib_here.*] -# ignore_missing_imports = True +[mypy-pyfuse3.*] +ignore_missing_imports = True + +[mypy-pyfuse3_asyncio.*] +ignore_missing_imports = True diff --git a/requirements-test.txt b/requirements-test.txt --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,2 +1,3 @@ pytest requests # workaround for https://forge.softwareheritage.org/T2634 +requests-mock diff --git a/requirements.txt b/requirements.txt --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ pyfuse3 +python-daemon diff --git a/swh/fuse/cache.py b/swh/fuse/cache.py new file mode 100644 --- /dev/null +++ b/swh/fuse/cache.py @@ -0,0 +1,150 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from abc import ABC +from contextlib import closing +import json +import sqlite3 +from typing import Any, Dict, Optional, Set + +from swh.model.identifiers import SWHID, parse_swhid +from swh.web.client.client import typify + + +class FuseCache: + """ SWH FUSE retrieves both metadata and file contents from the Software + Heritage archive via the network. In order to obtain reasonable performances + several caches are used to minimize network transfer. + + Caches are stored on disk in SQLite databases located at + `$XDG_CACHE_HOME/swh/fuse/`. + + All caches are persistent (i.e., they survive the restart of the SWH FUSE + process) and global (i.e., they are shared by concurrent SWH FUSE + processes). + + We assume that no cache *invalidation* is necessary, due to intrinsic + properties of the Software Heritage archive, such as integrity verification + and append-only archive changes. To clean the caches one can just remove the + corresponding files from disk. """ + + def __init__(self, cache_conf: Dict[str, Any]): + self.cache_conf = cache_conf + + def __enter__(self): + self.metadata = MetadataCache(self.cache_conf["metadata"]) + self.metadata.__enter__() + self.blob = BlobCache(self.cache_conf["blob"]) + self.blob.__enter__() + return self + + def __exit__(self, type=None, val=None, tb=None) -> None: + self.metadata.__exit__() + self.blob.__exit__() + + def get_cached_swhids(self) -> Set[SWHID]: + """ Return a list of all previously cached SWHID """ + + with closing(self.metadata.conn.cursor()) as metadata_cursor, ( + closing(self.blob.conn.cursor()) + ) as blob_cursor: + # Some entries can be in one cache but not in the other so create a + # set from all caches + metadata_cursor.execute("select swhid from metadata_cache") + blob_cursor.execute("select swhid from blob_cache") + swhids = metadata_cursor.fetchall() + blob_cursor.fetchall() + swhids = [parse_swhid(x[0]) for x in swhids] + return set(swhids) + + +class AbstractCache(ABC): + """ Abstract cache implementation to share common behavior between cache + types (such as: YAML config parsing, SQLite context manager) """ + + def __init__(self, conf: Dict[str, Any]): + self.conf = conf + + def __enter__(self): + # In-memory (thus temporary) caching is useful for testing purposes + if self.conf.get("in-memory", False): + path = ":memory:" + else: + path = self.conf["path"] + self.conn = sqlite3.connect(path) + return self + + def __exit__(self, type=None, val=None, tb=None) -> None: + self.conn.close() + + +class MetadataCache(AbstractCache): + """ The metadata cache map each SWHID to the complete metadata of the + referenced object. This is analogous to what is available in + `meta/.json` file (and generally used as data source for returning + the content of those files). """ + + def __enter__(self): + super().__enter__() + with self.conn as conn: + conn.execute("create table if not exists metadata_cache (swhid, metadata)") + return self + + def __getitem__(self, swhid: SWHID) -> Any: + with self.conn as conn, closing(conn.cursor()) as cursor: + cursor.execute( + "select metadata from metadata_cache where swhid=?", (str(swhid),) + ) + cache = cursor.fetchone() + if cache: + metadata = json.loads(cache[0]) + return typify(metadata, swhid.object_type) + else: + return None + + def __setitem__(self, swhid: SWHID, metadata: Any) -> None: + with self.conn as conn: + conn.execute( + "insert into metadata_cache values (?, ?)", + ( + str(swhid), + json.dumps( + metadata, + # Converts the typified JSON to plain str version + default=lambda x: ( + x.object_id if isinstance(x, SWHID) else x.__dict__ + ), + ), + ), + ) + + +class BlobCache(AbstractCache): + """ The blob cache map SWHIDs of type `cnt` to the bytes of their archived + content. + + The blob cache entry for a given content object is populated, at the latest, + the first time the object is `read()`-d. It might be populated earlier on + due to prefetching, e.g., when a directory pointing to the given content is + listed for the first time. """ + + def __enter__(self): + super().__enter__() + with self.conn as conn: + conn.execute("create table if not exists blob_cache (swhid, blob)") + return self + + def __getitem__(self, swhid: SWHID) -> Optional[str]: + with self.conn as conn, closing(conn.cursor()) as cursor: + cursor.execute("select blob from blob_cache where swhid=?", (str(swhid),)) + cache = cursor.fetchone() + if cache: + blob = cache[0] + return blob + else: + return None + + def __setitem__(self, swhid: SWHID, blob: str) -> None: + with self.conn as conn: + conn.execute("insert into blob_cache values (?, ?)", (str(swhid), blob)) diff --git a/swh/fuse/cli.py b/swh/fuse/cli.py --- a/swh/fuse/cli.py +++ b/swh/fuse/cli.py @@ -3,29 +3,65 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information +from contextlib import ExitStack +import os + # WARNING: do not import unnecessary things here to keep cli startup time under # control -import os -from typing import Any, Dict +from pathlib import Path +from typing import Any, Dict, Tuple import click +from daemon import DaemonContext # from swh.core import config from swh.core.cli import CONTEXT_SETTINGS +from swh.model.identifiers import SWHID # All generic config code should reside in swh.core.config DEFAULT_CONFIG_PATH = os.environ.get( "SWH_CONFIG_FILE", os.path.join(click.get_app_dir("swh"), "global.yml") ) -DEFAULT_CONFIG: Dict[str, Any] = { - "web-api": { - "url": "https://archive.softwareheritage.org/api/1", - "auth-token": None, - } +CACHE_HOME_DIR: Path = ( + Path(os.environ["XDG_CACHE_HOME"]) + if "XDG_CACHE_HOME" in os.environ + else Path.home() / ".cache" +) + +DEFAULT_CONFIG: Dict[str, Tuple[str, Any]] = { + "cache": ( + "dict", + { + "metadata": {"path": CACHE_HOME_DIR / "swh/fuse/metadata.sqlite"}, + "blob": {"path": CACHE_HOME_DIR / "swh/fuse/blob.sqlite"}, + }, + ), + "web-api": ( + "dict", + {"url": "https://archive.softwareheritage.org/api/1", "auth-token": None,}, + ), } +class SWHIDParamType(click.ParamType): + """Click argument that accepts SWHID and return them as + :class:`swh.model.identifiers.SWHID` instances + + """ + + name = "SWHID" + + def convert(self, value, param, ctx) -> SWHID: + from swh.model.exceptions import ValidationError + from swh.model.identifiers import parse_swhid + + try: + return parse_swhid(value) + except ValidationError: + self.fail(f'"{value}" is not a valid SWHID', param, ctx) + + @click.group(name="fuse", context_settings=CONTEXT_SETTINGS) # XXX conffile logic temporarily commented out due to: # XXX https://forge.softwareheritage.org/T2632 @@ -50,17 +86,36 @@ @cli.command() +@click.argument( + "path", + required=True, + metavar="PATH", + type=click.Path(exists=True, dir_okay=True, file_okay=False), +) +@click.argument("swhids", nargs=-1, metavar="[SWHID]...", type=SWHIDParamType()) @click.option( - "-u", - "--api-url", - default=DEFAULT_CONFIG["web-api"]["url"], - metavar="API_URL", + "--config-file", + "-C", + default=None, + type=click.Path(exists=True, dir_okay=False,), + help="YAML configuration file", +) +@click.option( + "-f", + "--foreground", + is_flag=True, show_default=True, - help="base URL for Software Heritage Web API", + help="Run FUSE system in foreground instead of daemon", ) @click.pass_context -def mount(ctx, api_url): - """Mount the Software Heritage archive at the given mount point""" - from .fuse import fuse # XXX +def mount(ctx, swhids, path, config_file, foreground): + """ Mount the Software Heritage archive at the given mount point """ + + from swh.core import config + from swh.fuse import fuse - fuse() + conf = config.read(config_file, DEFAULT_CONFIG) + with ExitStack() as stack: + if not foreground: + stack.enter(DaemonContext()) + fuse.main(swhids, path, conf) diff --git a/swh/fuse/fs/artifact.py b/swh/fuse/fs/artifact.py new file mode 100644 --- /dev/null +++ b/swh/fuse/fs/artifact.py @@ -0,0 +1,49 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Any, Dict, Iterator, List + +from swh.fuse.fs.entry import ArtifactEntry, VirtualEntry + + +class Content: + """ Software Heritage content artifact. + + Content leaves (AKA blobs) are represented on disks as regular files, + containing the corresponding bytes, as archived. + + Note that permissions are associated to blobs only in the context of + directories. Hence, when accessing blobs from the top-level `archive/` + directory, the permissions of the `archive/SWHID` file will be arbitrary and + not meaningful (e.g., `0x644`). """ + + def __init__(self, json: Dict[str, Any]): + self.json = json + + +class Directory: + """ Software Heritage directory artifact. + + Directory nodes are represented as directories on the file-system, + containing one entry for each entry of the archived directory. Entry names + and other metadata, including permissions, will correspond to the archived + entry metadata. + + Note that the FUSE mount is read-only, no matter what the permissions say. + So it is possible that, in the context of a directory, a file is presented + as writable, whereas actually writing to it will fail with `EPERM`. """ + + def __init__(self, json: List[Dict[str, Any]]): + self.json = json + + def __iter__(self) -> Iterator[VirtualEntry]: + entries = [] + for entry in self.json: + name, swhid = entry["name"], entry["target"] + # The directory API has extra info we can use to set attributes + # without additional Software Heritage API call + prefetch = entry + entries.append(ArtifactEntry(name, swhid, prefetch)) + return iter(entries) diff --git a/swh/fuse/fs/entry.py b/swh/fuse/fs/entry.py new file mode 100644 --- /dev/null +++ b/swh/fuse/fs/entry.py @@ -0,0 +1,55 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from enum import IntEnum +from stat import S_IFDIR, S_IFREG +from typing import Any, Dict + +from swh.model.identifiers import SWHID + + +class EntryMode(IntEnum): + """ Default entry mode and permissions for the FUSE. + + The FUSE mount is always read-only, even if permissions contradict this + statement (in a context of a directory, entries are listed with permissions + taken from the archive). + """ + + RDONLY_FILE = S_IFREG | 0o444 + RDONLY_DIR = S_IFDIR | 0o555 + + +class VirtualEntry: + """ Main wrapper class to manipulate virtual FUSE entries + + Attributes: + name: entry filename + mode: entry permission mode + """ + + def __init__(self, name: str, mode: EntryMode): + self.name = name + self.mode = mode + + +class ArtifactEntry(VirtualEntry): + """ FUSE virtual entry for a Software Heritage Artifact + + Attributes: + name: entry filename + swhid: Software Heritage persistent identifier + prefetch: optional prefetched metadata used to set entry attributes + """ + + def __init__(self, name: str, swhid: SWHID, prefetch: Dict[str, Any] = None): + self.name = name + self.swhid = swhid + self.prefetch = prefetch + + +ROOT_DIRENTRY = VirtualEntry("root", EntryMode.RDONLY_DIR) +ARCHIVE_DIRENTRY = VirtualEntry("archive", EntryMode.RDONLY_DIR) +META_DIRENTRY = VirtualEntry("meta", EntryMode.RDONLY_DIR) diff --git a/swh/fuse/fs/mountpoint.py b/swh/fuse/fs/mountpoint.py new file mode 100644 --- /dev/null +++ b/swh/fuse/fs/mountpoint.py @@ -0,0 +1,56 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from typing import Iterator + +from swh.fuse.cache import FuseCache +from swh.fuse.fs.entry import ( + ARCHIVE_DIRENTRY, + META_DIRENTRY, + ArtifactEntry, + EntryMode, + VirtualEntry, +) + + +class Root: + """ The FUSE mountpoint, consisting of the archive/ and meta/ directories """ + + def __iter__(self) -> Iterator[VirtualEntry]: + entries = [ARCHIVE_DIRENTRY, META_DIRENTRY] + return iter(entries) + + +class Archive: + """ The archive/ directory is lazily populated with one entry per accessed + SWHID, having actual SWHIDs as names """ + + def __init__(self, cache: FuseCache): + self.cache = cache + + def __iter__(self) -> Iterator[VirtualEntry]: + entries = [] + for swhid in self.cache.get_cached_swhids(): + entries.append(ArtifactEntry(str(swhid), swhid)) + return iter(entries) + + +class Meta: + """ The meta/ directory contains one SWHID.json file for each SWHID entry + under archive/. The JSON file contain all available meta information about + the given SWHID, as returned by the Software Heritage Web API for that + object. Note that, in case of pagination (e.g., snapshot objects with many + branches) the JSON file will contain a complete version with all pages + merged together. """ + + def __init__(self, cache: FuseCache): + self.cache = cache + + def __iter__(self) -> Iterator[VirtualEntry]: + entries = [] + for swhid in self.cache.get_cached_swhids(): + filename = str(swhid) + ".json" + entries.append(VirtualEntry(filename, EntryMode.RDONLY_FILE)) + return iter(entries) diff --git a/swh/fuse/fuse.py b/swh/fuse/fuse.py --- a/swh/fuse/fuse.py +++ b/swh/fuse/fuse.py @@ -3,11 +3,331 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information - +import asyncio +import errno +import itertools import logging -import sys +import os +from pathlib import Path +import time +from typing import Any, Dict, List + +import pyfuse3 +import pyfuse3_asyncio +import requests + +from swh.fuse.cache import FuseCache +from swh.fuse.fs.artifact import Content, Directory +from swh.fuse.fs.entry import ( + ARCHIVE_DIRENTRY, + META_DIRENTRY, + ROOT_DIRENTRY, + ArtifactEntry, + EntryMode, + VirtualEntry, +) +from swh.fuse.fs.mountpoint import Archive, Meta, Root +from swh.model.identifiers import CONTENT, DIRECTORY, SWHID, parse_swhid +from swh.web.client.client import WebAPIClient + + +class Fuse(pyfuse3.Operations): + """ Software Heritage Filesystem in Userspace (FUSE). Locally mount parts of + the archive and naviguate it as a virtual file system. """ + + def __init__( + self, + swhids: List[SWHID], + root_path: Path, + cache: FuseCache, + conf: Dict[str, Any], + ): + super(Fuse, self).__init__() + + self._next_inode: int = pyfuse3.ROOT_INODE + self._next_fd: int = 0 + + root_inode = self._next_inode + self._next_inode += 1 + + self._inode2entry: Dict[int, VirtualEntry] = {root_inode: ROOT_DIRENTRY} + self._entry2inode: Dict[VirtualEntry, int] = {ROOT_DIRENTRY: root_inode} + self._entry2fd: Dict[VirtualEntry, int] = {} + self._fd2entry: Dict[int, VirtualEntry] = {} + self._inode2path: Dict[int, Path] = {root_inode: root_path} + + self.time_ns: int = time.time_ns() # start time, used as timestamp + self.gid = os.getgid() + self.uid = os.getuid() + + self.web_api = WebAPIClient( + conf["web-api"]["url"], conf["web-api"]["auth-token"] + ) + self.cache = cache + + # Initially populate the cache + for swhid in swhids: + self.get_metadata(swhid) + + def shutdown(self) -> None: + pass + + def _alloc_inode(self, entry: VirtualEntry) -> int: + """ Return a unique inode integer for a given entry """ + + try: + return self._entry2inode[entry] + except KeyError: + inode = self._next_inode + self._next_inode += 1 + self._entry2inode[entry] = inode + self._inode2entry[inode] = entry + + # TODO add inode recycling with invocation to invalidate_inode when + # the dicts get too big + + return inode + + def _alloc_fd(self, entry: VirtualEntry) -> int: + """ Return a unique file descriptor integer for a given entry """ + + try: + return self._entry2fd[entry] + except KeyError: + fd = self._next_fd + self._next_fd += 1 + self._entry2fd[entry] = fd + self._fd2entry[fd] = entry + return fd + + def inode2entry(self, inode: int) -> VirtualEntry: + """ Return the entry matching a given inode """ + + try: + return self._inode2entry[inode] + except KeyError: + raise pyfuse3.FUSEError(errno.ENOENT) + + def entry2inode(self, entry: VirtualEntry) -> int: + """ Return the inode matching a given entry """ + + try: + return self._entry2inode[entry] + except KeyError: + raise pyfuse3.FUSEError(errno.ENOENT) + + def inode2path(self, inode: int) -> Path: + """ Return the path matching a given inode """ + + try: + return self._inode2path[inode] + except KeyError: + raise pyfuse3.FUSEError(errno.ENOENT) + + def get_metadata(self, swhid: SWHID) -> Any: + """ Retrieve metadata for a given SWHID using Software Heritage API """ + + # TODO: swh-graph API + cache = self.cache.metadata[swhid] + if cache: + return cache + + try: + metadata = self.web_api.get(swhid) + self.cache.metadata[swhid] = metadata + return metadata + except requests.HTTPError: + logging.error(f"Unknown SWHID: '{swhid}'") + + def get_blob(self, swhid: SWHID) -> str: + """ Retrieve the blob bytes for a given content SWHID using Software + Heritage API """ + + if swhid.object_type != CONTENT: + raise pyfuse3.FUSEError(errno.EINVAL) + + cache = self.cache.blob[swhid] + if cache: + return cache + + resp = list(self.web_api.content_raw(swhid)) + blob = "".join(map(bytes.decode, resp)) + self.cache.blob[swhid] = blob + return blob + + def get_direntries(self, entry: VirtualEntry) -> Any: + """ Return directory entries of a given entry """ + + if isinstance(entry, ArtifactEntry): + if entry.swhid.object_type == CONTENT: + raise pyfuse3.FUSEError(errno.ENOTDIR) + + metadata = self.get_metadata(entry.swhid) + if entry.swhid.object_type == CONTENT: + return Content(metadata) + if entry.swhid.object_type == DIRECTORY: + return Directory(metadata) + # TODO: add other objects + else: + if entry == ROOT_DIRENTRY: + return Root() + elif entry == ARCHIVE_DIRENTRY: + return Archive(self.cache) + elif entry == META_DIRENTRY: + return Meta(self.cache) + # TODO: error handling + + def get_attrs(self, entry: VirtualEntry) -> pyfuse3.EntryAttributes: + """ Return entry attributes """ + + attrs = pyfuse3.EntryAttributes() + attrs.st_size = 0 + attrs.st_atime_ns = self.time_ns + attrs.st_ctime_ns = self.time_ns + attrs.st_mtime_ns = self.time_ns + attrs.st_gid = self.gid + attrs.st_uid = self.uid + attrs.st_ino = self._alloc_inode(entry) + + if isinstance(entry, ArtifactEntry): + metadata = entry.prefetch or self.get_metadata(entry.swhid) + if entry.swhid.object_type == CONTENT: + # Only in the context of a directory entry do we have archived + # permissions. Otherwise, fallback to default read-only. + attrs.st_mode = metadata.get("perms", int(EntryMode.RDONLY_FILE)) + attrs.st_size = metadata["length"] + else: + attrs.st_mode = int(EntryMode.RDONLY_DIR) + else: + attrs.st_mode = int(entry.mode) + # Meta JSON entries (under the root meta/ directory) + if entry.name.endswith(".json"): + swhid = parse_swhid(entry.name.replace(".json", "")) + metadata = self.get_metadata(swhid) + attrs.st_size = len(str(metadata)) + + return attrs + + async def getattr( + self, inode: int, _ctx: pyfuse3.RequestContext + ) -> pyfuse3.EntryAttributes: + """ Get attributes for a given inode """ + + entry = self.inode2entry(inode) + return self.get_attrs(entry) + + async def opendir(self, inode: int, _ctx: pyfuse3.RequestContext) -> int: + """ Open a directory referred by a given inode """ + + # Re-use inode as directory handle + return inode + + async def readdir( + self, inode: int, offset: int, token: pyfuse3.ReaddirToken + ) -> None: + """ Read entries in an open directory """ + + direntry = self.inode2entry(inode) + path = self.inode2path(inode) + + # TODO: add cache on direntry list? + entries = self.get_direntries(direntry) + next_id = offset + 1 + for entry in itertools.islice(entries, offset, None): + name = os.fsencode(entry.name) + attrs = self.get_attrs(entry) + if not pyfuse3.readdir_reply(token, name, attrs, next_id): + break + + next_id += 1 + self._inode2entry[attrs.st_ino] = entry + self._inode2path[attrs.st_ino] = Path(path, entry.name) + + async def open( + self, inode: int, _flags: int, _ctx: pyfuse3.RequestContext + ) -> pyfuse3.FileInfo: + """ Open an inode and return a unique file descriptor """ + + entry = self.inode2entry(inode) + fd = self._alloc_fd(entry) + return pyfuse3.FileInfo(fh=fd, keep_cache=True) + + async def read(self, fd: int, _offset: int, _length: int) -> bytes: + """ Read blob content pointed by the given `fd` (file descriptor). Both + parameters `_offset` and `_length` are ignored. """ + # TODO: use offset/length + + try: + entry = self._fd2entry[fd] + except KeyError: + raise pyfuse3.FUSEError(errno.ENOENT) + + if isinstance(entry, ArtifactEntry): + blob = self.get_blob(entry.swhid) + return blob.encode() + else: + # Meta JSON entries (under the root meta/ directory) + if entry.name.endswith(".json"): + swhid = parse_swhid(entry.name.replace(".json", "")) + metadata = self.get_metadata(swhid) + return str(metadata).encode() + else: + # TODO: error handling + raise pyfuse3.FUSEError(errno.ENOENT) + + async def lookup( + self, parent_inode: int, name: str, _ctx: pyfuse3.RequestContext + ) -> pyfuse3.EntryAttributes: + """ Look up a directory entry by name and get its attributes """ + + name = os.fsdecode(name) + path = Path(self.inode2path(parent_inode), name) + parent_entry = self.inode2entry(parent_inode) + + attr = None + if isinstance(parent_entry, ArtifactEntry): + metadata = self.get_metadata(parent_entry.swhid) + for entry in metadata: + if entry["name"] == name: + swhid = entry["target"] + attr = self.get_attrs(ArtifactEntry(name, swhid)) + # TODO: this is fragile, maybe cache attrs? + else: + if parent_entry == ROOT_DIRENTRY: + if name == ARCHIVE_DIRENTRY.name: + attr = self.get_attrs(ARCHIVE_DIRENTRY) + elif name == META_DIRENTRY.name: + attr = self.get_attrs(META_DIRENTRY) + else: + swhid = parse_swhid(name) + attr = self.get_attrs(ArtifactEntry(name, swhid)) + + if attr: + self._inode2path[attr.st_ino] = path + return attr + else: + # TODO: error handling (name not found) + return pyfuse3.EntryAttributes() + + +def main(swhids: List[SWHID], root_path: Path, conf: Dict[str, Any]) -> None: + """ swh-fuse CLI entry-point """ + + # Use pyfuse3 asyncio layer to match the rest of Software Heritage codebase + pyfuse3_asyncio.enable() + + with FuseCache(conf["cache"]) as cache: + fs = Fuse(swhids, root_path, cache, conf) + fuse_options = set(pyfuse3.default_options) + fuse_options.add("fsname=swhfs") + fuse_options.add("debug") + pyfuse3.init(fs, root_path, fuse_options) -def fuse(): - logging.error("not implemented: FUSE mounting") - sys.exit(1) + loop = asyncio.get_event_loop() + try: + loop.run_until_complete(pyfuse3.main()) + fs.shutdown() + finally: + pyfuse3.close(unmount=True) + loop.close() diff --git a/swh/fuse/tests/api_data.py b/swh/fuse/tests/api_data.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/api_data.py @@ -0,0 +1,445 @@ +# GENERATED FILE, DO NOT EDIT. +# Run './gen-api-data.py > api_data.py' instead. +# fmt: off + +API_URL = 'https://invalid-test-only.archive.softwareheritage.org/api/1' +ROOT_SWHID = 'swh:1:dir:9eb62ef7dd283f7385e7d31af6344d9feedd25de' +ROOT_URL = 'directory/9eb62ef7dd283f7385e7d31af6344d9feedd25de/' +README_URL = 'content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/' +README_RAW_URL = 'content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/raw/' + +MOCK_ARCHIVE = { + 'directory/9eb62ef7dd283f7385e7d31af6344d9feedd25de/': # NoQA: E501 + r"""[ + { + "checksums": { + "sha1": "39a0a88cd8eae4504e1d33b0c0f88059044d761f", + "sha1_git": "a0a96088c74f49a961a80bc0851a84214b0a9f83", + "sha256": "7c0f4eaf45838f26ae951b490beb0d11034a30b21e5c39a54c4223f5c2018890" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 16166, + "name": ".clang-format", + "perms": 33188, + "status": "visible", + "target": "a0a96088c74f49a961a80bc0851a84214b0a9f83", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:a0a96088c74f49a961a80bc0851a84214b0a9f83/", + "type": "file" + }, + { + "checksums": { + "sha1": "0e31de4130c64f23e9ee5fc761fdbd807dc94360", + "sha1_git": "43967c6b20151ee126db08e24758e3c789bcb844", + "sha256": "dbd64d3f532b962d4681d79077cc186340f5f439de7f99c709b01892332af866" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 59, + "name": ".cocciconfig", + "perms": 33188, + "status": "visible", + "target": "43967c6b20151ee126db08e24758e3c789bcb844", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:43967c6b20151ee126db08e24758e3c789bcb844/", + "type": "file" + }, + { + "checksums": { + "sha1": "52b62d115dfae2ed19561db14e8d10ee22659e7f", + "sha1_git": "a64d219137455f407a7b1f2c6b156c5575852e9e", + "sha256": "4c9ba8e0ef521ce01474e98eddfc77afaec8a8e259939a139590c00505646527" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 71, + "name": ".get_maintainer.ignore", + "perms": 33188, + "status": "visible", + "target": "a64d219137455f407a7b1f2c6b156c5575852e9e", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:a64d219137455f407a7b1f2c6b156c5575852e9e/", + "type": "file" + }, + { + "checksums": { + "sha1": "6cc5a38e6f6ca93e21c78cb9c54794a42c3031c3", + "sha1_git": "4b32eaa9571e64e47b51c43537063f56b204d8b3", + "sha256": "dc52a4e1ee3615c87691aca7f667c7e49f6900f36b5c20339ac497366ba9406c" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 62, + "name": ".gitattributes", + "perms": 33188, + "status": "visible", + "target": "4b32eaa9571e64e47b51c43537063f56b204d8b3", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:4b32eaa9571e64e47b51c43537063f56b204d8b3/", + "type": "file" + }, + { + "checksums": { + "sha1": "1966a794db7d9518f321a8ecb0736c16de59bd91", + "sha1_git": "162bd2b67bdf6a28be7a361b8418e4e31d542854", + "sha256": "a9766c936a81df2ed3ea41f506ddaad88a78d9cf47e093df414b3b6f2e6d8e14" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 1852, + "name": ".gitignore", + "perms": 33188, + "status": "visible", + "target": "162bd2b67bdf6a28be7a361b8418e4e31d542854", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:162bd2b67bdf6a28be7a361b8418e4e31d542854/", + "type": "file" + }, + { + "checksums": { + "sha1": "2bb15bd51c981842b6f710d2e057972e5f22cfcc", + "sha1_git": "332c7833057f51da02805add9b60161ff31aee71", + "sha256": "d7b69571529964b3c8444a73ac720bbb883cc70fc4b78a36d1ac277f660c50bb" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 17283, + "name": ".mailmap", + "perms": 33188, + "status": "visible", + "target": "332c7833057f51da02805add9b60161ff31aee71", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:332c7833057f51da02805add9b60161ff31aee71/", + "type": "file" + }, + { + "checksums": { + "sha1": "0473e748fee37c7b68487fb102c0d563bbc641b3", + "sha1_git": "a635a38ef9405fdfcfe97f3a435393c1e9cae971", + "sha256": "fb5a425bd3b3cd6071a3a9aff9909a859e7c1158d54d32e07658398cd67eb6a0" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 496, + "name": "COPYING", + "perms": 33188, + "status": "visible", + "target": "a635a38ef9405fdfcfe97f3a435393c1e9cae971", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:a635a38ef9405fdfcfe97f3a435393c1e9cae971/", + "type": "file" + }, + { + "checksums": { + "sha1": "f27043aefa4b69b921df3728ee906c3f03087d29", + "sha1_git": "32ee70a7562eec7345e98841473abb438379a4fd", + "sha256": "23242c7183ee2815e27fea2346b0e5ad9131b8b611b200ae0418d4027cea2a3d" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 99788, + "name": "CREDITS", + "perms": 33188, + "status": "visible", + "target": "32ee70a7562eec7345e98841473abb438379a4fd", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:32ee70a7562eec7345e98841473abb438379a4fd/", + "type": "file" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "Documentation", + "perms": 16384, + "target": "1ba46735273aa020a173c0ad0c813179530dd117", + "target_url": "https://archive.softwareheritage.org/api/1/directory/1ba46735273aa020a173c0ad0c813179530dd117/", + "type": "dir" + }, + { + "checksums": { + "sha1": "2491dd3bed10f6918ed1657ab5a7a8efbddadf5d", + "sha1_git": "fa441b98c9f6eac1617acf1772ae8b371cfd42aa", + "sha256": "75df66064f75e91e6458862cd9413b19e65b77eefcc8a95dcbd6bf36fd2e4b59" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 1327, + "name": "Kbuild", + "perms": 33188, + "status": "visible", + "target": "fa441b98c9f6eac1617acf1772ae8b371cfd42aa", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:fa441b98c9f6eac1617acf1772ae8b371cfd42aa/", + "type": "file" + }, + { + "checksums": { + "sha1": "6b9b12a6bbff219dfbb45ec068af9aa7cf1b7288", + "sha1_git": "745bc773f567067a85ce6574fb41ce80833247d9", + "sha256": "a592dae7d067cd8e5dc43e3f9dc363eba9eb1f7cf80c6178b5cd291c0b76d3ec" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 555, + "name": "Kconfig", + "perms": 33188, + "status": "visible", + "target": "745bc773f567067a85ce6574fb41ce80833247d9", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:745bc773f567067a85ce6574fb41ce80833247d9/", + "type": "file" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "LICENSES", + "perms": 16384, + "target": "a49a894ea3684b6c044448c37f812356550d14a2", + "target_url": "https://archive.softwareheritage.org/api/1/directory/a49a894ea3684b6c044448c37f812356550d14a2/", + "type": "dir" + }, + { + "checksums": { + "sha1": "eb207b62f2fe0225dc55d9b87d82f6009e864117", + "sha1_git": "f0068bceeb6158a30c6eee430ca6d2a7e4c4013a", + "sha256": "3c81b34eaf99d943e4c2fac2548f3d9d740a9e9683ccedbaacfb82796c7965e1" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 569101, + "name": "MAINTAINERS", + "perms": 33188, + "status": "visible", + "target": "f0068bceeb6158a30c6eee430ca6d2a7e4c4013a", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:f0068bceeb6158a30c6eee430ca6d2a7e4c4013a/", + "type": "file" + }, + { + "checksums": { + "sha1": "9bfa205a4d23b9a60889bd9b59010574670b8b90", + "sha1_git": "f2116815416091dbfa7dcf58ae179ae3241ec1b1", + "sha256": "e87fb2b9482b9066b47c1656e55ebc4897bbb226daa122bcd4a2858fef19e597" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 63305, + "name": "Makefile", + "perms": 33188, + "status": "visible", + "target": "f2116815416091dbfa7dcf58ae179ae3241ec1b1", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:f2116815416091dbfa7dcf58ae179ae3241ec1b1/", + "type": "file" + }, + { + "checksums": { + "sha1": "ca1dc365022dcaa728dfb11bcde40ad3cce0574b", + "sha1_git": "669ac7c32292798644b21dbb5a0dc657125f444d", + "sha256": "bad58d396f62102befaf23a8a2ab6b1693fdc8f318de3059b489781f28865612" + }, + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": 727, + "name": "README", + "perms": 33188, + "status": "visible", + "target": "669ac7c32292798644b21dbb5a0dc657125f444d", + "target_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/", + "type": "file" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "arch", + "perms": 16384, + "target": "cf12c1ce4de958ab4ddcb008fe89118b82a3c7b7", + "target_url": "https://archive.softwareheritage.org/api/1/directory/cf12c1ce4de958ab4ddcb008fe89118b82a3c7b7/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "block", + "perms": 16384, + "target": "a77c89fa64b8ec37c9aa0fa98add54bfb6075257", + "target_url": "https://archive.softwareheritage.org/api/1/directory/a77c89fa64b8ec37c9aa0fa98add54bfb6075257/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "certs", + "perms": 16384, + "target": "527d8f94235029c6f571414df5f8ed2951a0ca5b", + "target_url": "https://archive.softwareheritage.org/api/1/directory/527d8f94235029c6f571414df5f8ed2951a0ca5b/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "crypto", + "perms": 16384, + "target": "1fb1357e2d22af4332091937ed960a47f78d0b5e", + "target_url": "https://archive.softwareheritage.org/api/1/directory/1fb1357e2d22af4332091937ed960a47f78d0b5e/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "drivers", + "perms": 16384, + "target": "3b5be1ee0216ec59c70e132681be4a5d79e7da9b", + "target_url": "https://archive.softwareheritage.org/api/1/directory/3b5be1ee0216ec59c70e132681be4a5d79e7da9b/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "fs", + "perms": 16384, + "target": "1dbf8d211613db72f5b83b0987023bd5acf866ee", + "target_url": "https://archive.softwareheritage.org/api/1/directory/1dbf8d211613db72f5b83b0987023bd5acf866ee/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "include", + "perms": 16384, + "target": "74991fd1a983c6b3f72c8815f7de81a3abddb255", + "target_url": "https://archive.softwareheritage.org/api/1/directory/74991fd1a983c6b3f72c8815f7de81a3abddb255/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "init", + "perms": 16384, + "target": "c944a589113271d878e27bbc31ae369edecaff90", + "target_url": "https://archive.softwareheritage.org/api/1/directory/c944a589113271d878e27bbc31ae369edecaff90/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "ipc", + "perms": 16384, + "target": "ff553b9398fea6b2e290ea4a95f7a94f1cf3c22c", + "target_url": "https://archive.softwareheritage.org/api/1/directory/ff553b9398fea6b2e290ea4a95f7a94f1cf3c22c/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "kernel", + "perms": 16384, + "target": "8c700fd3589e6d2befa4d9b2cc79471eac37da38", + "target_url": "https://archive.softwareheritage.org/api/1/directory/8c700fd3589e6d2befa4d9b2cc79471eac37da38/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "lib", + "perms": 16384, + "target": "0f2936da43bebe4f26b3be83e8fa392c4f9e82cf", + "target_url": "https://archive.softwareheritage.org/api/1/directory/0f2936da43bebe4f26b3be83e8fa392c4f9e82cf/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "mm", + "perms": 16384, + "target": "e15d954c1ed09e6fc29c184515834696d8e70e7c", + "target_url": "https://archive.softwareheritage.org/api/1/directory/e15d954c1ed09e6fc29c184515834696d8e70e7c/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "net", + "perms": 16384, + "target": "41e1603b37542d265eade0555e0db66668135575", + "target_url": "https://archive.softwareheritage.org/api/1/directory/41e1603b37542d265eade0555e0db66668135575/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "samples", + "perms": 16384, + "target": "9fa649fea3c8ab6b4926f0e7721a21a36b685153", + "target_url": "https://archive.softwareheritage.org/api/1/directory/9fa649fea3c8ab6b4926f0e7721a21a36b685153/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "scripts", + "perms": 16384, + "target": "e4e5b45d7c44d0bd2c6feb1a257fff7303d2c67e", + "target_url": "https://archive.softwareheritage.org/api/1/directory/e4e5b45d7c44d0bd2c6feb1a257fff7303d2c67e/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "security", + "perms": 16384, + "target": "a4a58d89fc506c3660610105a08de60614cdc980", + "target_url": "https://archive.softwareheritage.org/api/1/directory/a4a58d89fc506c3660610105a08de60614cdc980/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "sound", + "perms": 16384, + "target": "bf9e1568b8ce61157a322fddbaab1a0c76be15ef", + "target_url": "https://archive.softwareheritage.org/api/1/directory/bf9e1568b8ce61157a322fddbaab1a0c76be15ef/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "tools", + "perms": 16384, + "target": "83d6279411023bf7edf6bde6ce2e3748912f4936", + "target_url": "https://archive.softwareheritage.org/api/1/directory/83d6279411023bf7edf6bde6ce2e3748912f4936/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "usr", + "perms": 16384, + "target": "aae2ca939e0f7ac6b5e489e4c7835e1a15588cff", + "target_url": "https://archive.softwareheritage.org/api/1/directory/aae2ca939e0f7ac6b5e489e4c7835e1a15588cff/", + "type": "dir" + }, + { + "dir_id": "9eb62ef7dd283f7385e7d31af6344d9feedd25de", + "length": null, + "name": "virt", + "perms": 16384, + "target": "d7f6f10a8509839e404d1cc5af51317ac8b26276", + "target_url": "https://archive.softwareheritage.org/api/1/directory/d7f6f10a8509839e404d1cc5af51317ac8b26276/", + "type": "dir" + } +] +""", # NoQA: E501 + 'content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/': # NoQA: E501 + r"""{ + "checksums": { + "blake2s256": "746aaa0816ffc8cadf5e7f70b8bb93a47a76299ef263c743dbfef2644c6a0245", + "sha1": "ca1dc365022dcaa728dfb11bcde40ad3cce0574b", + "sha1_git": "669ac7c32292798644b21dbb5a0dc657125f444d", + "sha256": "bad58d396f62102befaf23a8a2ab6b1693fdc8f318de3059b489781f28865612" + }, + "data_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/raw/", + "filetype_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/filetype/", + "language_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/language/", + "length": 727, + "license_url": "https://archive.softwareheritage.org/api/1/content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/license/", + "status": "visible" +} +""", # NoQA: E501 + 'content/sha1_git:669ac7c32292798644b21dbb5a0dc657125f444d/raw/': # NoQA: E501 + r"""Linux kernel +============ + +There are several guides for kernel developers and users. These guides can +be rendered in a number of formats, like HTML and PDF. Please read +Documentation/admin-guide/README.rst first. + +In order to build the documentation, use ``make htmldocs`` or +``make pdfdocs``. The formatted documentation can also be read online at: + + https://www.kernel.org/doc/html/latest/ + +There are various text files in the Documentation/ subdirectory, +several of them using the Restructured Text markup notation. + +Please read the Documentation/process/changes.rst file, as it contains the +requirements for building and running the kernel, and information about +the problems which may result by upgrading your kernel. +""", # NoQA: E501 +} diff --git a/swh/fuse/tests/conftest.py b/swh/fuse/tests/conftest.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/conftest.py @@ -0,0 +1,65 @@ +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +from multiprocessing import Process +from os import listdir +from pathlib import Path +import subprocess +from tempfile import NamedTemporaryFile, TemporaryDirectory +import time + +from click.testing import CliRunner +import pytest +import yaml + +from swh.fuse import cli + +from .api_data import API_URL, MOCK_ARCHIVE, ROOT_SWHID + + +@pytest.fixture +def web_api_mock(requests_mock): + for api_call, data in MOCK_ARCHIVE.items(): + requests_mock.get(f"{API_URL}/{api_call}", text=data) + return requests_mock + + +@pytest.fixture +def fuse_mntdir(web_api_mock): + tmpdir = TemporaryDirectory(suffix=".swh-fuse-test") + tmpfile = NamedTemporaryFile(suffix=".swh-fuse-test.yml") + + config = { + "cache": {"metadata": {"in-memory": True}, "blob": {"in-memory": True}}, + "web-api": {"url": API_URL, "auth-token": None}, + } + + # Run FUSE in foreground mode but in a separate process, so it does not + # block execution and remains easy to kill during teardown + def fuse_process(tmpdir, tmpfile): + with tmpdir as mntdir, tmpfile as config_path: + config_path = Path(config_path.name) + config_path.write_text(yaml.dump(config)) + CliRunner().invoke( + cli.mount, + args=[mntdir, ROOT_SWHID, "--foreground", "--config-file", config_path], + ) + + fuse = Process(target=fuse_process, args=[tmpdir, tmpfile]) + fuse.start() + # Wait max 3 seconds for the FUSE to correctly mount + i = 0 + while i < 30: + try: + root = listdir(tmpdir.name) + if root: + break + except FileNotFoundError: + i += 1 + time.sleep(0.1) + + yield tmpdir.name + + subprocess.run(["fusermount", "-u", tmpdir.name], check=True) diff --git a/swh/fuse/tests/gen-api-data.py b/swh/fuse/tests/gen-api-data.py new file mode 100755 --- /dev/null +++ b/swh/fuse/tests/gen-api-data.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2020 The Software Heritage developers +# See the AUTHORS file at the top-level directory of this distribution +# License: GNU General Public License version 3, or any later version +# See top-level LICENSE file for more information + +import json + +import requests + +API_URL_real = "https://archive.softwareheritage.org/api/1" +API_URL_test = "https://invalid-test-only.archive.softwareheritage.org/api/1" + +# Use the Linux kernel as a testing repository +ROOT_HASH = "9eb62ef7dd283f7385e7d31af6344d9feedd25de" +README_HASH = "669ac7c32292798644b21dbb5a0dc657125f444d" + +ROOT_SWHID = f"swh:1:dir:{ROOT_HASH}" + +urls = { + "ROOT": f"directory/{ROOT_HASH}/", + "README": f"content/sha1_git:{README_HASH}/", + "README_RAW": f"content/sha1_git:{README_HASH}/raw/", +} + +print("# GENERATED FILE, DO NOT EDIT.") +print("# Run './gen-api-data.py > api_data.py' instead.") +print("# fmt: off") + +print("") +print(f"API_URL = '{API_URL_test}'") +print(f"ROOT_SWHID = '{ROOT_SWHID}'") +for name, url in urls.items(): + print(f"{name}_URL = '{url}'") +print("") + +print("MOCK_ARCHIVE = {") +for url in urls.values(): + print(f" '{url}': # NoQA: E501") + print(' r"""', end="") + + data = requests.get(f"{API_URL_real}/{url}").text + if url.endswith("/raw/"): + print(data, end="") + else: + parsed = json.loads(data) + print(json.dumps(parsed, indent=2, sort_keys=True)) + + print('""", # NoQA: E501') +print("}") diff --git a/swh/fuse/tests/test_cli.py b/swh/fuse/tests/test_cli.py --- a/swh/fuse/tests/test_cli.py +++ b/swh/fuse/tests/test_cli.py @@ -3,17 +3,15 @@ # License: GNU General Public License version 3, or any later version # See top-level LICENSE file for more information -import unittest +from pathlib import Path -from click.testing import CliRunner +from .api_data import ROOT_SWHID -from swh.fuse import cli - -class TestMount(unittest.TestCase): - def setUp(self): - self.runner = CliRunner() - - def test_no_args(self): - result = self.runner.invoke(cli.mount) - self.assertNotEqual(result.exit_code, 0) +def test_mountpoint(fuse_mntdir): + archive_dir = Path(fuse_mntdir, "archive") + meta_dir = Path(fuse_mntdir, "meta") + swhid_dir = Path(fuse_mntdir, "archive", ROOT_SWHID) + assert archive_dir.is_dir() + assert meta_dir.is_dir() + assert swhid_dir.is_dir() diff --git a/swh/fuse/tests/test_content.py b/swh/fuse/tests/test_content.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/test_content.py @@ -0,0 +1,16 @@ +from pathlib import Path + +from .api_data import MOCK_ARCHIVE, README_RAW_URL, ROOT_SWHID + + +def test_file_exists(fuse_mntdir): + readme_path = Path(fuse_mntdir, "archive", ROOT_SWHID, "README") + assert readme_path.is_file() + + +def test_cat_file(fuse_mntdir): + readme_path = Path(fuse_mntdir, "archive", ROOT_SWHID, "README") + expected = MOCK_ARCHIVE[README_RAW_URL] + with open(readme_path, "r") as f: + actual = f.read() + assert actual == expected diff --git a/swh/fuse/tests/test_directory.py b/swh/fuse/tests/test_directory.py new file mode 100644 --- /dev/null +++ b/swh/fuse/tests/test_directory.py @@ -0,0 +1,14 @@ +import json +from os import listdir +from pathlib import Path + +from .api_data import MOCK_ARCHIVE, ROOT_SWHID, ROOT_URL + + +def test_ls_root_swhid(fuse_mntdir): + root_resp = json.loads(MOCK_ARCHIVE[ROOT_URL]) + expected = [entry["name"] for entry in root_resp] + + swhid_dir = Path(fuse_mntdir, "archive", ROOT_SWHID) + actual = listdir(swhid_dir) + assert actual == expected