Changeset View
Changeset View
Standalone View
Standalone View
swh/fuse/cache.py
Show All 13 Lines | |||||
from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple | from typing import Any, AsyncGenerator, Dict, List, Optional, Tuple | ||||
import aiosqlite | import aiosqlite | ||||
import dateutil.parser | import dateutil.parser | ||||
from psutil import virtual_memory | from psutil import virtual_memory | ||||
from swh.fuse.fs.artifact import RevisionHistoryShardByDate | from swh.fuse.fs.artifact import RevisionHistoryShardByDate | ||||
from swh.fuse.fs.entry import FuseDirEntry, FuseEntry | from swh.fuse.fs.entry import FuseDirEntry, FuseEntry | ||||
from swh.fuse.fs.mountpoint import ArchiveDir, OriginDir | from swh.fuse.fs.mountpoint import CacheDir, OriginDir | ||||
from swh.model.exceptions import ValidationError | from swh.model.exceptions import ValidationError | ||||
from swh.model.identifiers import REVISION, SWHID, parse_swhid | from swh.model.identifiers import REVISION, SWHID, parse_swhid | ||||
from swh.web.client.client import ORIGIN_VISIT, typify_json | from swh.web.client.client import ORIGIN_VISIT, typify_json | ||||
class FuseCache: | class FuseCache: | ||||
"""SwhFS retrieves both metadata and file contents from the Software Heritage archive | """SwhFS retrieves both metadata and file contents from the Software Heritage archive | ||||
via the network. In order to obtain reasonable performances several caches are used | via the network. In order to obtain reasonable performances several caches are used | ||||
▲ Show 20 Lines • Show All 329 Lines • ▼ Show 20 Lines | def __init__(self, conf: Dict[str, Any]): | ||||
max_ram = int(float(num) * units[unit]) | max_ram = int(float(num) * units[unit]) | ||||
self.lru_cache = self.LRU(max_ram) | self.lru_cache = self.LRU(max_ram) | ||||
def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | ||||
return self.lru_cache.get(direntry.inode, None) | return self.lru_cache.get(direntry.inode, None) | ||||
def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | ||||
if isinstance(direntry, (ArchiveDir, OriginDir)): | if isinstance(direntry, (CacheDir, OriginDir)): | ||||
# The `archive/`, and `origin/` are populated on the fly so we | # The `cache/` and `origin/` directories are populated on the fly | ||||
# should never cache them | |||||
pass | pass | ||||
elif ( | elif ( | ||||
isinstance(direntry, RevisionHistoryShardByDate) | isinstance(direntry, RevisionHistoryShardByDate) | ||||
and not direntry.is_status_done | and not direntry.is_status_done | ||||
): | ): | ||||
# The `by-date/' directory is populated in parallel so only cache it | # The `by-date/' directory is populated in parallel so only cache it | ||||
# once it has finished fetching all data from the API | # once it has finished fetching all data from the API | ||||
pass | pass | ||||
else: | else: | ||||
self.lru_cache[direntry.inode] = entries | self.lru_cache[direntry.inode] = entries |