Changeset View
Changeset View
Standalone View
Standalone View
swh/web/api/views/graph.py
# Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from distutils.util import strtobool | from distutils.util import strtobool | ||||
import json | import json | ||||
from typing import Dict | from typing import Dict, Iterator, Union | ||||
import requests | import requests | ||||
from django.http.response import StreamingHttpResponse | |||||
from rest_framework.decorators import renderer_classes | from rest_framework.decorators import renderer_classes | ||||
from rest_framework.request import Request | from rest_framework.request import Request | ||||
from rest_framework.response import Response | from rest_framework.response import Response | ||||
from swh.model.identifiers import ORIGIN, parse_swhid | from swh.model.identifiers import ORIGIN, parse_swhid | ||||
from swh.web.api.apidoc import api_doc | from swh.web.api.apidoc import api_doc | ||||
from swh.web.api.apiurls import api_route | from swh.web.api.apiurls import api_route | ||||
from swh.web.api.renderers import PlainTextRenderer | from swh.web.api.renderers import PlainTextRenderer | ||||
Show All 17 Lines | if parsed_swhid.object_type == ORIGIN: | ||||
)[0] | )[0] | ||||
assert origin_info is not None | assert origin_info is not None | ||||
origin_urls[parsed_swhid.object_id] = origin_info["url"] | origin_urls[parsed_swhid.object_id] = origin_info["url"] | ||||
return origin_info["url"] | return origin_info["url"] | ||||
else: | else: | ||||
return swhid | return swhid | ||||
def _resolve_origin_swhids_in_graph_response(response: requests.Response) -> str: | def _resolve_origin_swhids_in_graph_response( | ||||
response: requests.Response, | |||||
) -> Iterator[bytes]: | |||||
""" | """ | ||||
Resolve origin urls from their swhid sha1 representations in graph service | Resolve origin urls from their swhid sha1 representations in graph service | ||||
responses. | responses. | ||||
""" | """ | ||||
content_type = response.headers["Content-Type"] | content_type = response.headers["Content-Type"] | ||||
origin_urls: Dict[str, str] = {} | origin_urls: Dict[str, str] = {} | ||||
if content_type == "application/x-ndjson": | if content_type == "application/x-ndjson": | ||||
processed_response = [] | for line in response.iter_lines(): | ||||
for line in response.text.split("\n")[:-1]: | swhids = json.loads(line.decode("utf-8")) | ||||
swhids = json.loads(line) | |||||
processed_line = [] | processed_line = [] | ||||
for swhid in swhids: | for swhid in swhids: | ||||
processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | ||||
processed_response.append(json.dumps(processed_line)) | yield (json.dumps(processed_line) + "\n").encode() | ||||
return "\n".join(processed_response) + "\n" | |||||
elif content_type == "text/plain": | elif content_type == "text/plain": | ||||
processed_response = [] | for line in response.iter_lines(): | ||||
for line in response.text.split("\n")[:-1]: | |||||
processed_line = [] | processed_line = [] | ||||
swhids = line.split(" ") | swhids = line.decode("utf-8").split(" ") | ||||
for swhid in swhids: | for swhid in swhids: | ||||
processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | ||||
processed_response.append(" ".join(processed_line)) | yield (" ".join(processed_line) + "\n").encode() | ||||
return "\n".join(processed_response) + "\n" | else: | ||||
return response.text | for line in response.iter_lines(): | ||||
yield line + b"\n" | |||||
@api_route(r"/graph/", "api-1-graph-doc") | @api_route(r"/graph/", "api-1-graph-doc") | ||||
@api_doc("/graph/") | @api_doc("/graph/") | ||||
def api_graph(request: Request) -> None: | def api_graph(request: Request) -> None: | ||||
""" | """ | ||||
.. http:get:: /api/1/graph/(graph_query)/ | .. http:get:: /api/1/graph/(graph_query)/ | ||||
Show All 37 Lines | .. http:get:: /api/1/graph/(graph_query)/ | ||||
:swh_web_api:`graph/visit/paths/swh:1:dir:644dd466d8ad527ea3a609bfd588a3244e6dafcb?direction=backward&resolve_origins=true` | :swh_web_api:`graph/visit/paths/swh:1:dir:644dd466d8ad527ea3a609bfd588a3244e6dafcb?direction=backward&resolve_origins=true` | ||||
""" | """ | ||||
return None | return None | ||||
@api_route(r"/graph/(?P<graph_query>.+)/", "api-1-graph") | @api_route(r"/graph/(?P<graph_query>.+)/", "api-1-graph") | ||||
@renderer_classes([PlainTextRenderer]) | @renderer_classes([PlainTextRenderer]) | ||||
def api_graph_proxy(request: Request, graph_query: str) -> Response: | def api_graph_proxy( | ||||
request: Request, graph_query: str | |||||
) -> Union[Response, StreamingHttpResponse]: | |||||
if request.get_host() != SWH_WEB_INTERNAL_SERVER_NAME: | if request.get_host() != SWH_WEB_INTERNAL_SERVER_NAME: | ||||
if not bool(request.user and request.user.is_authenticated): | if not bool(request.user and request.user.is_authenticated): | ||||
return Response("Authentication credentials were not provided.", status=401) | return Response("Authentication credentials were not provided.", status=401) | ||||
if not request.user.has_perm(API_GRAPH_PERM): | if not request.user.has_perm(API_GRAPH_PERM): | ||||
return Response( | return Response( | ||||
"You do not have permission to perform this action.", status=403 | "You do not have permission to perform this action.", status=403 | ||||
) | ) | ||||
graph_query_url = get_config()["graph"]["server_url"] | graph_query_url = get_config()["graph"]["server_url"] | ||||
graph_query_url += graph_query | graph_query_url += graph_query | ||||
if request.GET: | if request.GET: | ||||
graph_query_url += "?" + request.GET.urlencode(safe="/;:") | graph_query_url += "?" + request.GET.urlencode(safe="/;:") | ||||
response = requests.get(graph_query_url) | response = requests.get(graph_query_url, stream=True) | ||||
response_text = response.text | # graph stats and counter endpoint responses are not streamed | ||||
if response.headers.get("Transfer-Encoding") != "chunked": | |||||
return Response( | |||||
response.text, | |||||
status=response.status_code, | |||||
content_type=response.headers["Content-Type"], | |||||
) | |||||
# other endpoint responses are streamed | |||||
else: | |||||
resolve_origins = strtobool(request.GET.get("resolve_origins", "false")) | resolve_origins = strtobool(request.GET.get("resolve_origins", "false")) | ||||
if response.status_code == 200 and resolve_origins: | if response.status_code == 200 and resolve_origins: | ||||
response_text = _resolve_origin_swhids_in_graph_response(response) | response_stream = _resolve_origin_swhids_in_graph_response(response) | ||||
return Response( | else: | ||||
response_text, | response_stream = map(lambda line: line + b"\n", response.iter_lines()) | ||||
return StreamingHttpResponse( | |||||
response_stream, | |||||
status=response.status_code, | status=response.status_code, | ||||
content_type=response.headers["Content-Type"], | content_type=response.headers["Content-Type"], | ||||
) | ) |