diff --git a/swh/graph/backend.py b/swh/graph/backend.py
--- a/swh/graph/backend.py
+++ b/swh/graph/backend.py
@@ -42,7 +42,7 @@
         self.graph_path = graph_path
         self.config = check_config(config or {})
 
-    def __enter__(self):
+    def start_gateway(self):
         self.gateway = JavaGateway.launch_gateway(
             java_path=None,
             javaopts=self.config["java_tool_options"].split(),
@@ -56,10 +56,16 @@
         self.node2swhid = NodeToSwhidMap(self.graph_path + "." + NODE2SWHID_EXT)
         self.swhid2node = SwhidToNodeMap(self.graph_path + "." + SWHID2NODE_EXT)
         self.stream_proxy = JavaStreamProxy(self.entry)
+
+    def stop_gateway(self):
+        self.gateway.shutdown()
+
+    def __enter__(self):
+        self.start_gateway()
         return self
 
     def __exit__(self, exc_type, exc_value, tb):
-        self.gateway.shutdown()
+        self.stop_gateway()
 
     def stats(self):
         return self.entry.stats()
diff --git a/swh/graph/cli.py b/swh/graph/cli.py
--- a/swh/graph/cli.py
+++ b/swh/graph/cli.py
@@ -339,14 +339,11 @@
     """run the graph RPC service"""
     import aiohttp
 
-    from swh.graph.backend import Backend
     from swh.graph.server.app import make_app
 
-    backend = Backend(graph_path=graph, config=ctx.obj["config"])
-    app = make_app(backend=backend)
+    app = make_app(config=ctx.obj["config"])
 
-    with backend:
-        aiohttp.web.run_app(app, host=host, port=port)
+    aiohttp.web.run_app(app, host=host, port=port)
 
 
 @graph_cli_group.command()
diff --git a/swh/graph/server/app.py b/swh/graph/server/app.py
--- a/swh/graph/server/app.py
+++ b/swh/graph/server/app.py
@@ -11,11 +11,14 @@
 import asyncio
 from collections import deque
 import json
+import os
 from typing import Optional
 
 import aiohttp.web
 
 from swh.core.api.asynchronous import RPCServerApp
+from swh.core.config import read as config_read
+from swh.graph.backend import Backend
 from swh.model.exceptions import ValidationError
 from swh.model.identifiers import EXTENDED_SWHID_TYPES
 
@@ -30,6 +33,23 @@
 RANDOM_RETRIES = 5  # TODO make this configurable via rpc-serve configuration
 
 
+class GraphServerApp(RPCServerApp):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.on_startup.append(self._start_gateway)
+        self.on_shutdown.append(self._stop_gateway)
+
+    @staticmethod
+    async def _start_gateway(app):
+        # Equivalent to entering `with app["backend"]:`
+        app["backend"].start_gateway()
+
+    @staticmethod
+    async def _stop_gateway(app):
+        # Equivalent to exiting `with app["backend"]:` with no error
+        app["backend"].stop_gateway()
+
+
 async def index(request):
     return aiohttp.web.Response(
         content_type="text/html",
@@ -334,8 +354,12 @@
     count_type = "visit_nodes"
 
 
-def make_app(backend, **kwargs):
-    app = RPCServerApp(**kwargs)
+def make_app(config=None, backend=None, **kwargs):
+    if (config is None) == (backend is None):
+        raise ValueError("make_app() expects exactly one of 'config' or 'backend'")
+    if backend is None:
+        backend = Backend(graph_path=config["graph"]["path"], config=config["graph"])
+    app = GraphServerApp(**kwargs)
     app.add_routes(
         [
             aiohttp.web.get("/", index),
@@ -357,3 +381,12 @@
 
     app["backend"] = backend
     return app
+
+
+def make_app_from_configfile():
+    """Load configuration and then build application to run
+
+    """
+    config_file = os.environ.get("SWH_CONFIG_FILENAME")
+    config = config_read(config_file)
+    return make_app(config=config)
diff --git a/swh/graph/tests/conftest.py b/swh/graph/tests/conftest.py
--- a/swh/graph/tests/conftest.py
+++ b/swh/graph/tests/conftest.py
@@ -28,14 +28,13 @@
     def run(self):
         try:
             backend = Backend(graph_path=str(TEST_GRAPH_PATH))
-            with backend:
-                with loop_context() as loop:
-                    app = make_app(backend=backend, debug=True)
-                    client = TestClient(TestServer(app), loop=loop)
-                    loop.run_until_complete(client.start_server())
-                    url = client.make_url("/graph/")
-                    self.q.put(url)
-                    loop.run_forever()
+            with loop_context() as loop:
+                app = make_app(backend=backend, debug=True)
+                client = TestClient(TestServer(app), loop=loop)
+                loop.run_until_complete(client.start_server())
+                url = client.make_url("/graph/")
+                self.q.put(url)
+                loop.run_forever()
         except Exception as e:
             self.q.put(e)