Changeset View
Changeset View
Standalone View
Standalone View
swh/graph/server/app.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
""" | """ | ||||
A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using | A proxy HTTP server for swh-graph, talking to the Java code via py4j, and using | ||||
FIFO as a transport to stream integers between the two languages. | FIFO as a transport to stream integers between the two languages. | ||||
""" | """ | ||||
import asyncio | import asyncio | ||||
from collections import deque | from collections import deque | ||||
import json | import json | ||||
import os | |||||
from typing import Optional | from typing import Optional | ||||
import aiohttp.web | import aiohttp.web | ||||
from swh.core.api.asynchronous import RPCServerApp | from swh.core.api.asynchronous import RPCServerApp | ||||
from swh.core.config import read as config_read | |||||
from swh.graph.backend import Backend | |||||
from swh.model.exceptions import ValidationError | from swh.model.exceptions import ValidationError | ||||
from swh.model.identifiers import EXTENDED_SWHID_TYPES | from swh.model.identifiers import EXTENDED_SWHID_TYPES | ||||
try: | try: | ||||
from contextlib import asynccontextmanager | from contextlib import asynccontextmanager | ||||
except ImportError: | except ImportError: | ||||
# Compatibility with 3.6 backport | # Compatibility with 3.6 backport | ||||
from async_generator import asynccontextmanager # type: ignore | from async_generator import asynccontextmanager # type: ignore | ||||
# maximum number of retries for random walks | # maximum number of retries for random walks | ||||
RANDOM_RETRIES = 5 # TODO make this configurable via rpc-serve configuration | RANDOM_RETRIES = 5 # TODO make this configurable via rpc-serve configuration | ||||
class GraphServerApp(RPCServerApp): | |||||
def __init__(self, *args, **kwargs): | |||||
super().__init__(*args, **kwargs) | |||||
self.on_startup.append(self._start_gateway) | |||||
self.on_shutdown.append(self._stop_gateway) | |||||
@staticmethod | |||||
async def _start_gateway(app): | |||||
# Equivalent to entering `with app["backend"]:` | |||||
app["backend"].start_gateway() | |||||
@staticmethod | |||||
async def _stop_gateway(app): | |||||
# Equivalent to exiting `with app["backend"]:` with no error | |||||
app["backend"].stop_gateway() | |||||
async def index(request): | async def index(request): | ||||
return aiohttp.web.Response( | return aiohttp.web.Response( | ||||
content_type="text/html", | content_type="text/html", | ||||
body="""<html> | body="""<html> | ||||
<head><title>Software Heritage storage server</title></head> | <head><title>Software Heritage storage server</title></head> | ||||
<body> | <body> | ||||
<p>You have reached the <a href="https://www.softwareheritage.org/"> | <p>You have reached the <a href="https://www.softwareheritage.org/"> | ||||
Software Heritage</a> graph API server.</p> | Software Heritage</a> graph API server.</p> | ||||
▲ Show 20 Lines • Show All 288 Lines • ▼ Show 20 Lines | |||||
class CountLeavesView(CountView): | class CountLeavesView(CountView): | ||||
count_type = "leaves" | count_type = "leaves" | ||||
class CountVisitNodesView(CountView): | class CountVisitNodesView(CountView): | ||||
count_type = "visit_nodes" | count_type = "visit_nodes" | ||||
def make_app(backend, **kwargs): | def make_app(config=None, backend=None, **kwargs): | ||||
app = RPCServerApp(**kwargs) | if (config is None) == (backend is None): | ||||
raise ValueError("make_app() expects exactly one of 'config' or 'backend'") | |||||
if backend is None: | |||||
backend = Backend(graph_path=config["graph"]["path"], config=config["graph"]) | |||||
app = GraphServerApp(**kwargs) | |||||
ardumont: Couldn't `backend` be a constructor argument of GraphServerApp to make clearer the collaborator… | |||||
vlorentzAuthorUnsubmitted Done Inline ActionsI don't know. I think @seirl wants to rewrite this, anyway vlorentz: I don't know.
I think @seirl wants to rewrite this, anyway | |||||
app.add_routes( | app.add_routes( | ||||
[ | [ | ||||
aiohttp.web.get("/", index), | aiohttp.web.get("/", index), | ||||
aiohttp.web.get("/graph", index), | aiohttp.web.get("/graph", index), | ||||
aiohttp.web.view("/graph/stats", StatsView), | aiohttp.web.view("/graph/stats", StatsView), | ||||
aiohttp.web.view("/graph/leaves/{src}", LeavesView), | aiohttp.web.view("/graph/leaves/{src}", LeavesView), | ||||
aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), | aiohttp.web.view("/graph/neighbors/{src}", NeighborsView), | ||||
aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), | aiohttp.web.view("/graph/visit/nodes/{src}", VisitNodesView), | ||||
aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), | aiohttp.web.view("/graph/visit/edges/{src}", VisitEdgesView), | ||||
aiohttp.web.view("/graph/visit/paths/{src}", VisitPathsView), | aiohttp.web.view("/graph/visit/paths/{src}", VisitPathsView), | ||||
# temporarily disabled in wait of a proper fix for T1969 | # temporarily disabled in wait of a proper fix for T1969 | ||||
# aiohttp.web.view("/graph/walk/{src}/{dst}", WalkView) | # aiohttp.web.view("/graph/walk/{src}/{dst}", WalkView) | ||||
aiohttp.web.view("/graph/randomwalk/{src}/{dst}", RandomWalkView), | aiohttp.web.view("/graph/randomwalk/{src}/{dst}", RandomWalkView), | ||||
aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), | aiohttp.web.view("/graph/neighbors/count/{src}", CountNeighborsView), | ||||
aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), | aiohttp.web.view("/graph/leaves/count/{src}", CountLeavesView), | ||||
aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), | aiohttp.web.view("/graph/visit/nodes/count/{src}", CountVisitNodesView), | ||||
] | ] | ||||
) | ) | ||||
app["backend"] = backend | app["backend"] = backend | ||||
return app | return app | ||||
def make_app_from_configfile(): | |||||
"""Load configuration and then build application to run | |||||
""" | |||||
config_file = os.environ.get("SWH_CONFIG_FILENAME") | |||||
config = config_read(config_file) | |||||
return make_app(config=config) |
Couldn't backend be a constructor argument of GraphServerApp to make clearer the collaborator relationship in the RPC class GraphServerApp?
(maybe not due to the @staticmethod in GraphServerApp though...)