diff --git a/swh/fuse/cache.py b/swh/fuse/cache.py --- a/swh/fuse/cache.py +++ b/swh/fuse/cache.py @@ -112,7 +112,8 @@ async def set(self, swhid: SWHID, metadata: Any) -> None: await self.conn.execute( "insert into metadata_cache values (?, ?)", - (str(swhid), json.dumps(metadata)), + # Keep the keys sorted so we can always retrieve them in the same order + (str(swhid), json.dumps(metadata, sort_keys=True)), ) await self.conn.commit() diff --git a/swh/fuse/fs/artifact.py b/swh/fuse/fs/artifact.py --- a/swh/fuse/fs/artifact.py +++ b/swh/fuse/fs/artifact.py @@ -71,7 +71,7 @@ async def __aiter__(self) -> AsyncIterator[FuseEntry]: metadata = await self.fuse.get_metadata(self.swhid) - for entry in metadata: + for entry in metadata[self.key]: name = entry["name"] swhid = entry["target"] mode = ( @@ -151,38 +151,50 @@ metadata = await self.fuse.get_metadata(self.swhid) directory = metadata["directory"] parents = metadata["parents"] + entries = [] root_path = self.get_relative_root_path() - yield self.create_child( - FuseSymlinkEntry, - name="root", - target=Path(root_path, f"archive/{directory}"), + entries.append( + self.create_child( + FuseSymlinkEntry, + name="root", + target=Path(root_path, f"archive/{directory}"), + ) ) - yield self.create_child( - FuseSymlinkEntry, - name="meta.json", - target=Path(root_path, f"meta/{self.swhid}.json"), + entries.append( + self.create_child( + FuseSymlinkEntry, + name="meta.json", + target=Path(root_path, f"meta/{self.swhid}.json"), + ) ) - yield self.create_child( - RevisionParents, - name="parents", - mode=int(EntryMode.RDONLY_DIR), - parents=[x["id"] for x in parents], + entries.append( + self.create_child( + RevisionParents, + name="parents", + mode=int(EntryMode.RDONLY_DIR), + parents=[x["id"] for x in parents], + ) ) if len(parents) >= 1: - yield self.create_child( - FuseSymlinkEntry, name="parent", target="parents/1/", + entries.append( + self.create_child(FuseSymlinkEntry, name="parent", target="parents/1/",) ) - yield self.create_child( - RevisionHistory, - name="history", - mode=int(EntryMode.RDONLY_DIR), - swhid=self.swhid, + entries.append( + self.create_child( + RevisionHistory, + name="history", + mode=int(EntryMode.RDONLY_DIR), + swhid=self.swhid, + ) ) + for entry in entries[self.key]: + yield entry + @dataclass class RevisionParents(FuseDirEntry): @@ -192,7 +204,7 @@ async def __aiter__(self) -> AsyncIterator[FuseEntry]: root_path = self.get_relative_root_path() - for i, parent in enumerate(self.parents): + for i, parent in enumerate(self.parents[self.key]): yield self.create_child( FuseSymlinkEntry, name=str(i + 1), @@ -209,7 +221,7 @@ async def __aiter__(self) -> AsyncIterator[FuseEntry]: history = await self.fuse.get_history(self.swhid) root_path = self.get_relative_root_path() - for swhid in history: + for swhid in history[self.key]: yield self.create_child( FuseSymlinkEntry, name=str(swhid), @@ -252,32 +264,46 @@ async def __aiter__(self) -> AsyncIterator[FuseEntry]: metadata = await self.fuse.get_metadata(self.swhid) root_path = self.get_relative_root_path() + entries = [] - yield self.create_child( - FuseSymlinkEntry, - name="meta.json", - target=Path(root_path, f"meta/{self.swhid}.json"), + entries.append( + self.create_child( + FuseSymlinkEntry, + name="meta.json", + target=Path(root_path, f"meta/{self.swhid}.json"), + ) ) target = metadata["target"] - yield self.create_child( - FuseSymlinkEntry, name="target", target=Path(root_path, f"archive/{target}") + entries.append( + self.create_child( + FuseSymlinkEntry, + name="target", + target=Path(root_path, f"archive/{target}"), + ) ) - yield self.create_child( - ReleaseType, - name="target_type", - mode=int(EntryMode.RDONLY_FILE), - target_type=target.object_type, + entries.append( + self.create_child( + ReleaseType, + name="target_type", + mode=int(EntryMode.RDONLY_FILE), + target_type=target.object_type, + ) ) target_dir = await self.find_root_directory(target) if target_dir is not None: - yield self.create_child( - FuseSymlinkEntry, - name="root", - target=Path(root_path, f"archive/{target_dir}"), + entries.append( + self.create_child( + FuseSymlinkEntry, + name="root", + target=Path(root_path, f"archive/{target_dir}"), + ) ) + for entry in entries[self.key]: + yield entry + @dataclass class ReleaseType(FuseFileEntry): @@ -308,9 +334,10 @@ async def __aiter__(self) -> AsyncIterator[FuseEntry]: metadata = await self.fuse.get_metadata(self.swhid) + metadata = list(metadata.items()) root_path = self.get_relative_root_path() - for branch_name, branch_meta in metadata.items(): + for (branch_name, branch_meta) in metadata[self.key]: # Mangle branch name to create a valid UNIX filename name = urllib.parse.quote_plus(branch_name) yield self.create_child( diff --git a/swh/fuse/fs/entry.py b/swh/fuse/fs/entry.py --- a/swh/fuse/fs/entry.py +++ b/swh/fuse/fs/entry.py @@ -5,7 +5,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, replace from enum import IntEnum from pathlib import Path from stat import S_IFDIR, S_IFLNK, S_IFREG @@ -57,7 +57,15 @@ return "../" * (self.depth - 1) def create_child(self, constructor: Any, **kwargs) -> FuseEntry: - return constructor(depth=self.depth + 1, fuse=self.fuse, **kwargs) + # TODO: this key field is really a pain to set because i cannot make a + # default value without conflicting with sub-types, and cannot create a + # field(init=False, default=...) as well because of the replace() call + try: + return constructor( + depth=self.depth + 1, fuse=self.fuse, key=slice(0, None), **kwargs + ) + except TypeError: + return constructor(depth=self.depth + 1, fuse=self.fuse, **kwargs) class FuseFileEntry(FuseEntry): @@ -69,12 +77,18 @@ raise NotImplementedError +@dataclass class FuseDirEntry(FuseEntry): """ FUSE virtual directory entry """ + key: Any + async def size(self) -> int: return 0 + def __getitem__(self, key: Any) -> Any: + return replace(self, key=key) + async def __aiter__(self): """ Return the child entries of a directory entry """ diff --git a/swh/fuse/fs/mountpoint.py b/swh/fuse/fs/mountpoint.py --- a/swh/fuse/fs/mountpoint.py +++ b/swh/fuse/fs/mountpoint.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field import json -from typing import AsyncIterator +from typing import Any, AsyncIterator from swh.fuse.fs.artifact import OBJTYPE_GETTERS from swh.fuse.fs.entry import EntryMode, FuseDirEntry, FuseEntry, FuseFileEntry @@ -20,10 +20,12 @@ name: str = field(init=False, default=None) mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) depth: int = field(init=False, default=1) + key: Any = field(default=slice(0, None)) async def __aiter__(self) -> AsyncIterator[FuseEntry]: - yield self.create_child(ArchiveDir) - yield self.create_child(MetaDir) + entries = [self.create_child(ArchiveDir), self.create_child(MetaDir)] + for entry in entries[self.key]: + yield entry @dataclass @@ -47,8 +49,12 @@ ) async def __aiter__(self) -> AsyncIterator[FuseEntry]: + entries = [] async for swhid in self.fuse.cache.get_cached_swhids(): - yield self.create_child(swhid) + entries.append(self.create_child(swhid)) + + for entry in entries[self.key]: + yield entry async def lookup(self, name: str) -> FuseEntry: entry = await super().lookup(name) @@ -77,14 +83,20 @@ mode: int = field(init=False, default=int(EntryMode.RDONLY_DIR)) async def __aiter__(self) -> AsyncIterator[FuseEntry]: + entries = [] async for swhid in self.fuse.cache.get_cached_swhids(): - yield self.create_child( - MetaEntry, - name=f"{swhid}.json", - mode=int(EntryMode.RDONLY_FILE), - swhid=swhid, + entries.append( + self.create_child( + MetaEntry, + name=f"{swhid}.json", + mode=int(EntryMode.RDONLY_FILE), + swhid=swhid, + ) ) + for entry in entries[self.key]: + yield entry + @dataclass class MetaEntry(FuseFileEntry): diff --git a/swh/fuse/fuse.py b/swh/fuse/fuse.py --- a/swh/fuse/fuse.py +++ b/swh/fuse/fuse.py @@ -174,13 +174,8 @@ direntry = self.inode2entry(inode) assert isinstance(direntry, FuseDirEntry) next_id = offset + 1 - i = 0 try: - async for entry in direntry: - if i < offset: - i += 1 - continue - + async for entry in direntry[offset:]: name = os.fsencode(entry.name) attrs = await self.get_attrs(entry) if not pyfuse3.readdir_reply(token, name, attrs, next_id): diff --git a/swh/fuse/tests/test_meta.py b/swh/fuse/tests/test_meta.py --- a/swh/fuse/tests/test_meta.py +++ b/swh/fuse/tests/test_meta.py @@ -12,5 +12,5 @@ file_path_meta = fuse_mntdir / f"meta/{swhid}.json" assert file_path_meta.exists() - expected = json.dumps(get_data_from_web_archive(swhid)) + expected = json.dumps(get_data_from_web_archive(swhid), sort_keys=True) assert file_path_meta.read_text() == expected diff --git a/swh/fuse/tests/test_release.py b/swh/fuse/tests/test_release.py --- a/swh/fuse/tests/test_release.py +++ b/swh/fuse/tests/test_release.py @@ -14,7 +14,7 @@ def test_access_meta(fuse_mntdir): file_path = fuse_mntdir / "archive" / ROOT_REL / "meta.json" - expected = json.dumps(get_data_from_web_archive(ROOT_REL)) + expected = json.dumps(get_data_from_web_archive(ROOT_REL), sort_keys=True) assert file_path.read_text() == expected diff --git a/swh/fuse/tests/test_revision.py b/swh/fuse/tests/test_revision.py --- a/swh/fuse/tests/test_revision.py +++ b/swh/fuse/tests/test_revision.py @@ -12,7 +12,7 @@ def test_access_meta(fuse_mntdir): file_path = fuse_mntdir / "archive" / ROOT_REV / "meta.json" - expected = json.dumps(get_data_from_web_archive(ROOT_REV)) + expected = json.dumps(get_data_from_web_archive(ROOT_REV), sort_keys=True) assert file_path.read_text() == expected