diff --git a/swh/fuse/fuse.py b/swh/fuse/fuse.py
--- a/swh/fuse/fuse.py
+++ b/swh/fuse/fuse.py
@@ -60,9 +60,6 @@
         self._next_inode += 1
         self._inode2entry[inode] = entry
 
-        # TODO add inode recycling with invocation to invalidate_inode when
-        # the dicts get too big
-
         return inode
 
     def _remove_inode(self, inode: int) -> None:
@@ -71,6 +68,11 @@
         except KeyError:
             pass
 
+        try:
+            pyfuse3.invalidate_inode(inode)
+        except FileNotFoundError:
+            pass
+
     def inode2entry(self, inode: int) -> FuseEntry:
         """ Return the entry matching a given inode """