Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/server/app.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| """ | """ | ||||
| A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using | A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using | ||||
| FIFO as a transport to stream integers between the two languages. | FIFO as a transport to stream integers between the two languages. | ||||
| """ | """ | ||||
| import asyncio | import asyncio | ||||
| from collections import deque | from collections import deque | ||||
| import json | import json | ||||
| import os | |||||
| from typing import Optional | from typing import Optional | ||||
| import aiohttp.web | import aiohttp.web | ||||
| from swh.core.api.asynchronous import RPCServerApp | from swh.core.api.asynchronous import RPCServerApp | ||||
| from swh.core.config import read as config_read | |||||
| from swh.graph.backend import Backend | |||||
| from swh.model.exceptions import ValidationError | from swh.model.exceptions import ValidationError | ||||
| from swh.model.identifiers import EXTENDED_SWHID_TYPES | from swh.model.identifiers import EXTENDED_SWHID_TYPES | ||||
| try: | try: | ||||
| from contextlib import asynccontextmanager | from contextlib import asynccontextmanager | ||||
| except ImportError: | except ImportError: | ||||
| # Compatibility with 3.6 backport | # Compatibility with 3.6 backport | ||||
| from async_generator import asynccontextmanager # type: ignore | from async_generator import asynccontextmanager # type: ignore | ||||
| # maximum number of retries for random walks | # maximum number of retries for random walks | ||||
| RANDOM_RETRIES = 5 # TODO make this configurable via rpc-serve configuration | RANDOM_RETRIES = 5 # TODO make this configurable via rpc-serve configuration | ||||
| class GraphServerApp(RPCServerApp): | |||||
| def __init__(self, *args, **kwargs): | |||||
| super().__init__(*args, **kwargs) | |||||
| self.on_startup.append(self._start_gateway) | |||||
| self.on_shutdown.append(self._stop_gateway) | |||||
| @staticmethod | |||||
| async def _start_gateway(app): | |||||
| # Equivalent to entering `with app["backend"]:` | |||||
| app["backend"].start_gateway() | |||||
| @staticmethod | |||||
| async def _stop_gateway(app): | |||||
| # Equivalent to exiting `with app["backend"]:` with no error | |||||
| app["backend"].stop_gateway() | |||||
| async def index(request): | async def index(request): | ||||
| return aiohttp.web.Response( | return aiohttp.web.Response( | ||||
| content_type="text/html", | content_type="text/html", | ||||
| body="""<html> | body="""<html> | ||||
| <head><title>Software Heritage storage server</title></head> | <head><title>Software Heritage storage server</title></head> | ||||
| <body> | <body> | ||||
| <p>You have reached the <a href="https://www.softwareheritage.org/"> | <p>You have reached the <a href="https://www.softwareheritage.org/"> | ||||
| Software Heritage</a> graph API server.</p> | Software Heritage</a> graph API server.</p> | ||||
| ▲ Show 20 Lines • Show All 288 Lines • ▼ Show 20 Lines | |||||
| class CountLeavesView(CountView): | class CountLeavesView(CountView): | ||||
| count_type = "leaves" | count_type = "leaves" | ||||
| class CountVisitNodesView(CountView): | class CountVisitNodesView(CountView): | ||||
| count_type = "visit_nodes" | count_type = "visit_nodes" | ||||
| def make_app(backend, **kwargs): | def make_app(config=None, backend=None, **kwargs): | ||||
| app = RPCServerApp(**kwargs) | if (config is None) == (backend is None): | ||||
| raise ValueError("make_app() expects exactly one of 'config' or 'backend'") | |||||
| if backend is None: | |||||
| backend = Backend(graph_path=config["graph"]["path"], config=config["graph"]) | |||||
ardumont: Couldn't `backend` be a constructor argument of GraphServerApp to make clearer the collaborator… | |||||
Done Inline ActionsI don't know. I think @seirl wants to rewrite this, anyway vlorentz: I don't know.
I think @seirl wants to rewrite this, anyway | |||||
| app = GraphServerApp(**kwargs) | |||||
| app.add_routes( | app.add_routes( | ||||
| [ | [ | ||||
| aiohttp.web.get("/", index), | aiohttp.web.get("/", index), | ||||
| aiohttp.web.get("/graph", index), | aiohttp.web.get("/graph", index), | ||||
| aiohttp.web.view("/graph/stats", StatsView), | aiohttp.web.view("/graph/stats", StatsView), | ||||
| aiohttp.web.view("/graph/leaves/{src}", LeavesView), | aiohttp.web.view("/graph/leaves/{src}", LeavesView), | ||||
| aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), | aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), | ||||
| aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), | aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), | ||||
| aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), | aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), | ||||
| aiohttp.web.view("/graph/visit/paths/{src}", VisitPathsView), | aiohttp.web.view("/graph/visit/paths/{src}", VisitPathsView), | ||||
| # temporarily disabled in wait of a proper fix for T1969 | # temporarily disabled in wait of a proper fix for T1969 | ||||
| # aiohttp.web.view("/graph/walk/{src}/{dst}", WalkView) | # aiohttp.web.view("/graph/walk/{src}/{dst}", WalkView) | ||||
| aiohttp.web.view("/graph/randomwalk/{src}/{dst}", RandomWalkView), | aiohttp.web.view("/graph/randomwalk/{src}/{dst}", RandomWalkView), | ||||
| aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), | aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), | ||||
| aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), | aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), | ||||
| aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), | aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), | ||||
| ] | ] | ||||
| ) | ) | ||||
| app["backend"] = backend | app["backend"] = backend | ||||
| return app | return app | ||||
| def make_app_from_configfile(): | |||||
| """Load configuration and then build application to run | |||||
| """ | |||||
| config_file = os.environ.get("SWH_CONFIG_FILENAME") | |||||
| config = config_read(config_file) | |||||
| return make_app(config=config) | |||||
Couldn't backend be a constructor argument of GraphServerApp to make clearer the collaborator relationship in the RPC class GraphServerApp?
(maybe not due to the @staticmethod in GraphServerApp though...)