Changeset View
Changeset View
Standalone View
Standalone View
swh/fuse/cache.py
| Show First 20 Lines • Show All 181 Lines • ▼ Show 20 Lines | class MetadataCache(AbstractCache): | ||||
| async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | async def set_visits(self, url_encoded: str, visits: List[Dict[str, Any]]) -> None: | ||||
| await self.conn.execute( | await self.conn.execute( | ||||
| "insert or replace into visits_cache values (?, ?, ?)", | "insert or replace into visits_cache values (?, ?, ?)", | ||||
| (url_encoded, json.dumps(visits), datetime.now()), | (url_encoded, json.dumps(visits), datetime.now()), | ||||
| ) | ) | ||||
| await self.conn.commit() | await self.conn.commit() | ||||
| async def remove(self, swhid: SWHID) -> None: | |||||
| await self.conn.execute( | |||||
| "delete from metadata_cache where swhid=?", (str(swhid),), | |||||
| ) | |||||
| await self.conn.commit() | |||||
| class BlobCache(AbstractCache): | class BlobCache(AbstractCache): | ||||
| """ The blob cache map SWHIDs of type `cnt` to the bytes of their archived | """ The blob cache map SWHIDs of type `cnt` to the bytes of their archived | ||||
| content. | content. | ||||
| The blob cache entry for a given content object is populated, at the latest, | The blob cache entry for a given content object is populated, at the latest, | ||||
| the first time the object is `read()`-d. It might be populated earlier on | the first time the object is `read()`-d. It might be populated earlier on | ||||
| due to prefetching, e.g., when a directory pointing to the given content is | due to prefetching, e.g., when a directory pointing to the given content is | ||||
| Show All 24 Lines | async def get(self, swhid: SWHID) -> Optional[bytes]: | ||||
| return None | return None | ||||
| async def set(self, swhid: SWHID, blob: bytes) -> None: | async def set(self, swhid: SWHID, blob: bytes) -> None: | ||||
| await self.conn.execute( | await self.conn.execute( | ||||
| "insert into blob_cache values (?, ?)", (str(swhid), blob) | "insert into blob_cache values (?, ?)", (str(swhid), blob) | ||||
| ) | ) | ||||
| await self.conn.commit() | await self.conn.commit() | ||||
| async def remove(self, swhid: SWHID) -> None: | |||||
| await self.conn.execute( | |||||
| "delete from blob_cache where swhid=?", (str(swhid),), | |||||
| ) | |||||
| await self.conn.commit() | |||||
| class HistoryCache(AbstractCache): | class HistoryCache(AbstractCache): | ||||
| """ The history cache map SWHIDs of type `rev` to a list of `rev` SWHIDs | """ The history cache map SWHIDs of type `rev` to a list of `rev` SWHIDs | ||||
| corresponding to all its revision ancestors, sorted in reverse topological | corresponding to all its revision ancestors, sorted in reverse topological | ||||
| order. As the parents cache, the history cache is lazily populated and can | order. As the parents cache, the history cache is lazily populated and can | ||||
| be prefetched. To efficiently store the ancestor lists, the history cache | be prefetched. To efficiently store the ancestor lists, the history cache | ||||
| represents ancestors as graph edges (a pair of two SWHID nodes), meaning the | represents ancestors as graph edges (a pair of two SWHID nodes), meaning the | ||||
| history cache is shared amongst all revisions parents. """ | history cache is shared amongst all revisions parents. """ | ||||
| ▲ Show 20 Lines • Show All 131 Lines • ▼ Show 20 Lines | def __init__(self, conf: Dict[str, Any]): | ||||
| max_ram = int(float(num) * units[unit]) | max_ram = int(float(num) * units[unit]) | ||||
| self.lru_cache = self.LRU(max_ram) | self.lru_cache = self.LRU(max_ram) | ||||
| def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | def get(self, direntry: FuseDirEntry) -> Optional[List[FuseEntry]]: | ||||
| return self.lru_cache.get(direntry.inode, None) | return self.lru_cache.get(direntry.inode, None) | ||||
| def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | def set(self, direntry: FuseDirEntry, entries: List[FuseEntry]) -> None: | ||||
| if isinstance(direntry, (CacheDir, OriginDir)): | if isinstance(direntry, (CacheDir, CacheDir.ArtifactShardBySwhid, OriginDir)): | ||||
| # The `cache/` and `origin/` directories are populated on the fly | # The `cache/` and `origin/` directories are populated on the fly | ||||
| pass | pass | ||||
| elif ( | elif ( | ||||
| isinstance(direntry, RevisionHistoryShardByDate) | isinstance(direntry, RevisionHistoryShardByDate) | ||||
| and not direntry.is_status_done | and not direntry.is_status_done | ||||
| ): | ): | ||||
| # The `by-date/' directory is populated in parallel so only cache it | # The `by-date/' directory is populated in parallel so only cache it | ||||
| # once it has finished fetching all data from the API | # once it has finished fetching all data from the API | ||||
| pass | pass | ||||
| else: | else: | ||||
| self.lru_cache[direntry.inode] = entries | self.lru_cache[direntry.inode] = entries | ||||