Changeset View
Changeset View
Standalone View
Standalone View
swh/fuse/cache.py
Show First 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | class MetadataCache(AbstractCache): | ||||
async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | ||||
await self.conn.execute( | await self.conn.execute( | ||||
"insert or replace into visits_cache values (?, ?, ?)", | "insert or replace into visits_cache values (?, ?, ?)", | ||||
(url_encoded, json.dumps(visits), datetime.now()), | (url_encoded, json.dumps(visits), datetime.now()), | ||||
) | ) | ||||
await self.conn.commit() | await self.conn.commit() | ||||
async def remove(self, swhid: SWHID) -> None: | |||||
await self.conn.execute( | |||||
"delete from metadata_cache where swhid=?", (str(swhid),), | |||||
) | |||||
await self.conn.commit() | |||||
class BlobCache(AbstractCache): | class BlobCache(AbstractCache): | ||||
""" The blob cache map SWHIDs of type `cnt` to the bytes of their archived | """ The blob cache map SWHIDs of type `cnt` to the bytes of their archived | ||||
content. | content. | ||||
The blob cache entry for a given content object is populated, at the latest, | The blob cache entry for a given content object is populated, at the latest, | ||||
the first time the object is `read()`-d. It might be populated earlier on | the first time the object is `read()`-d. It might be populated earlier on | ||||
due to prefetching, e.g., when a directory pointing to the given content is | due to prefetching, e.g., when a directory pointing to the given content is | ||||
Show All 24 Lines | async def get(self, swhid: SWHID) -> Optional[bytes]: | ||||
return None | return None | ||||
async def set(self, swhid: SWHID, blob: bytes) -> None: | async def set(self, swhid: SWHID, blob: bytes) -> None: | ||||
await self.conn.execute( | await self.conn.execute( | ||||
"insert into blob_cache values (?, ?)", (str(swhid), blob) | "insert into blob_cache values (?, ?)", (str(swhid), blob) | ||||
) | ) | ||||
await self.conn.commit() | await self.conn.commit() | ||||
async def remove(self, swhid: SWHID) -> None: | |||||
await self.conn.execute( | |||||
"delete from blob_cache where swhid=?", (str(swhid),), | |||||
) | |||||
await self.conn.commit() | |||||
class HistoryCache(AbstractCache): | class HistoryCache(AbstractCache): | ||||
""" The history cache map SWHIDs of type `rev` to a list of `rev` SWHIDs | """ The history cache map SWHIDs of type `rev` to a list of `rev` SWHIDs | ||||
corresponding to all its revision ancestors, sorted in reverse topological | corresponding to all its revision ancestors, sorted in reverse topological | ||||
order. As the parents cache, the history cache is lazily populated and can | order. As the parents cache, the history cache is lazily populated and can | ||||
be prefetched. To efficiently store the ancestor lists, the history cache | be prefetched. To efficiently store the ancestor lists, the history cache | ||||
represents ancestors as graph edges (a pair of two SWHID nodes), meaning the | represents ancestors as graph edges (a pair of two SWHID nodes), meaning the | ||||
history cache is shared amongst all revisions parents. """ | history cache is shared amongst all revisions parents. """ | ||||
▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def __init__(self, conf: Dict[str, Any]): | ||||
max_ram = int(float(num) * units[unit]) | max_ram = int(float(num) * units[unit]) | ||||
self.lru_cache = self.LRU(max_ram) | self.lru_cache = self.LRU(max_ram) | ||||
def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | ||||
return self.lru_cache.get(direntry.inode, None) | return self.lru_cache.get(direntry.inode, None) | ||||
def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | ||||
if isinstance(direntry, (CacheDir, OriginDir)): | if isinstance(direntry, (CacheDir, CacheDir.ArtifactShardBySwhid, OriginDir)): | ||||
# The `cache/` and `origin/` directories are populated on the fly | # The `cache/` and `origin/` directories are populated on the fly | ||||
pass | pass | ||||
elif ( | elif ( | ||||
isinstance(direntry, RevisionHistoryShardByDate) | isinstance(direntry, RevisionHistoryShardByDate) | ||||
and not direntry.is_status_done | and not direntry.is_status_done | ||||
): | ): | ||||
# The `by-date/' directory is populated in parallel so only cache it | # The `by-date/' directory is populated in parallel so only cache it | ||||
# once it has finished fetching all data from the API | # once it has finished fetching all data from the API | ||||
pass | pass | ||||
else: | else: | ||||
self.lru_cache[direntry.inode] = entries | self.lru_cache[direntry.inode] = entries |