diff --git a/swh/fuse/cache.py b/swh/fuse/cache.py
--- a/swh/fuse/cache.py
+++ b/swh/fuse/cache.py
@@ -49,14 +49,11 @@
         self.cache_conf = cache_conf
 
     async def __aenter__(self):
-        # History and raw metadata share the same SQLite db
-        self.metadata = MetadataCache(self.cache_conf["metadata"])
-        self.history = HistoryCache(self.cache_conf["metadata"])
-        self.blob = BlobCache(self.cache_conf["blob"])
+        self.metadata = await MetadataCache(self.cache_conf["metadata"]).__aenter__()
+        self.blob = await BlobCache(self.cache_conf["blob"]).__aenter__()
+        # History and raw metadata share the same SQLite db (hence the same connection)
+        self.history = await HistoryCache(self.metadata.conn).__aenter__()
         self.direntry = DirEntryCache(self.cache_conf["direntry"])
-        await self.metadata.__aenter__()
-        await self.blob.__aenter__()
-        await self.history.__aenter__()
         return self
 
     async def __aexit__(self, type=None, val=None, tb=None) -> None:
@@ -257,12 +254,20 @@
         create index if not exists idx_history on history_graph(src);
     """
 
+    def __init__(self, conn: sqlite3.Connection):
+        # The history cache is a bit particular because it shares the metadata
+        # SQLite db, thus they will share the connection as well to avoid locking
+        self.conn = conn
+
     async def __aenter__(self):
-        await super().__aenter__()
         await self.conn.executescript(self.DB_SCHEMA)
         await self.conn.commit()
         return self
 
+    async def __aexit__(self, type=None, val=None, tb=None) -> None:
+        # Do not close metadata db connection twice
+        pass
+
     HISTORY_REC_QUERY = """
         with recursive
         dfs(node) AS (