diff --git a/swh/fuse/cache.py b/swh/fuse/cache.py
--- a/swh/fuse/cache.py
+++ b/swh/fuse/cache.py
@@ -353,6 +353,10 @@
             self.move_to_end(key)
             return value
 
+        def __delitem__(self, key: Any) -> None:
+            self.used_ram -= self.sizeof(self[key])
+            super().__delitem__(key)
+
         def __setitem__(self, key: Any, value: Any) -> None:
             if key in self:
                 self.move_to_end(key)
@@ -363,7 +367,6 @@
 
             while self.used_ram > self.max_ram and self:
                 oldest = next(iter(self))
-                self.used_ram -= self.sizeof(oldest)
                 del self[oldest]
 
     def __init__(self, conf: Dict[str, Any]):
@@ -390,12 +393,11 @@
         if isinstance(direntry, (CacheDir, CacheDir.ArtifactShardBySwhid, OriginDir)):
             # The `cache/` and `origin/` directories are populated on the fly
             pass
-        elif (
-            isinstance(direntry, RevisionHistoryShardByDate)
-            and not direntry.is_status_done
-        ):
-            # The `by-date/' directory is populated in parallel so only cache it
-            # once it has finished fetching all data from the API
-            pass
         else:
             self.lru_cache[direntry.inode] = entries
+
+    def invalidate(self, direntry: FuseDirEntry) -> None:
+        try:
+            del self.lru_cache[direntry.inode]
+        except KeyError:
+            pass
diff --git a/swh/fuse/fs/artifact.py b/swh/fuse/fs/artifact.py
--- a/swh/fuse/fs/artifact.py
+++ b/swh/fuse/fs/artifact.py
@@ -220,22 +220,29 @@
 
     swhid: SWHID
 
-    async def prefill_caches(self) -> None:
+    async def prefill_by_date_cache(self, by_date_dir: FuseDirEntry) -> None:
         history = await self.fuse.get_history(self.swhid)
-        for swhid in history:
+        one_percent = max(round(0.01 * len(history)), 1)  # prevent modulo by 0
+        for idx, swhid in enumerate(history):
             await self.fuse.get_metadata(swhid)
+            # The by-date/ directory is cached temporarily in direntry, and
+            # invalidated + updated on a regular basis
+            if idx % one_percent == 0 or (idx + 1) == len(history):
+                self.fuse.cache.direntry.invalidate(by_date_dir)
 
     async def compute_entries(self) -> AsyncIterator[FuseEntry]:
-        # Run it concurrently because of the many API calls necessary
-        asyncio.create_task(self.prefill_caches())
-
-        yield self.create_child(
+        by_date_dir = self.create_child(
             RevisionHistoryShardByDate,
             name="by-date",
             mode=int(EntryMode.RDONLY_DIR),
             history_swhid=self.swhid,
         )
 
+        # Run it concurrently because of the many API calls necessary
+        asyncio.create_task(self.prefill_by_date_cache(by_date_dir))
+
+        yield by_date_dir
+
         yield self.create_child(
             RevisionHistoryShardByHash,
             name="by-hash",