Changeset View
Changeset View
Standalone View
Standalone View
swh/web/api/views/graph.py
| # Copyright (C) 2020 The Software Heritage developers | # Copyright (C) 2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU Affero General Public License version 3, or any later version | # License: GNU Affero General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from distutils.util import strtobool | from distutils.util import strtobool | ||||
| import json | import json | ||||
| from typing import Dict | from typing import Dict, Iterator, Union | ||||
| import requests | import requests | ||||
| from django.http.response import StreamingHttpResponse | |||||
| from rest_framework.decorators import renderer_classes | from rest_framework.decorators import renderer_classes | ||||
| from rest_framework.request import Request | from rest_framework.request import Request | ||||
| from rest_framework.response import Response | from rest_framework.response import Response | ||||
| from swh.model.identifiers import ORIGIN, parse_swhid | from swh.model.identifiers import ORIGIN, parse_swhid | ||||
| from swh.web.api.apidoc import api_doc | from swh.web.api.apidoc import api_doc | ||||
| from swh.web.api.apiurls import api_route | from swh.web.api.apiurls import api_route | ||||
| from swh.web.api.renderers import PlainTextRenderer | from swh.web.api.renderers import PlainTextRenderer | ||||
| Show All 17 Lines | if parsed_swhid.object_type == ORIGIN: | ||||
| )[0] | )[0] | ||||
| assert origin_info is not None | assert origin_info is not None | ||||
| origin_urls[parsed_swhid.object_id] = origin_info["url"] | origin_urls[parsed_swhid.object_id] = origin_info["url"] | ||||
| return origin_info["url"] | return origin_info["url"] | ||||
| else: | else: | ||||
| return swhid | return swhid | ||||
| def _resolve_origin_swhids_in_graph_response(response: requests.Response) -> str: | def _resolve_origin_swhids_in_graph_response( | ||||
| response: requests.Response, | |||||
| ) -> Iterator[bytes]: | |||||
| """ | """ | ||||
| Resolve origin urls from their swhid sha1 representations in graph service | Resolve origin urls from their swhid sha1 representations in graph service | ||||
| responses. | responses. | ||||
| """ | """ | ||||
| content_type = response.headers["Content-Type"] | content_type = response.headers["Content-Type"] | ||||
| origin_urls: Dict[str, str] = {} | origin_urls: Dict[str, str] = {} | ||||
| if content_type == "application/x-ndjson": | if content_type == "application/x-ndjson": | ||||
| processed_response = [] | for line in response.iter_lines(): | ||||
| for line in response.text.split("\n")[:-1]: | swhids = json.loads(line.decode("utf-8")) | ||||
| swhids = json.loads(line) | |||||
| processed_line = [] | processed_line = [] | ||||
| for swhid in swhids: | for swhid in swhids: | ||||
| processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | ||||
| processed_response.append(json.dumps(processed_line)) | yield (json.dumps(processed_line) + "\n").encode() | ||||
| return "\n".join(processed_response) + "\n" | |||||
| elif content_type == "text/plain": | elif content_type == "text/plain": | ||||
| processed_response = [] | for line in response.iter_lines(): | ||||
| for line in response.text.split("\n")[:-1]: | |||||
| processed_line = [] | processed_line = [] | ||||
| swhids = line.split(" ") | swhids = line.decode("utf-8").split(" ") | ||||
| for swhid in swhids: | for swhid in swhids: | ||||
| processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | processed_line.append(_resolve_origin_swhid(swhid, origin_urls)) | ||||
| processed_response.append(" ".join(processed_line)) | yield (" ".join(processed_line) + "\n").encode() | ||||
| return "\n".join(processed_response) + "\n" | else: | ||||
| return response.text | for line in response.iter_lines(): | ||||
| yield line + b"\n" | |||||
| @api_route(r"/graph/", "api-1-graph-doc") | @api_route(r"/graph/", "api-1-graph-doc") | ||||
| @api_doc("/graph/") | @api_doc("/graph/") | ||||
| def api_graph(request: Request) -> None: | def api_graph(request: Request) -> None: | ||||
| """ | """ | ||||
| .. http:get:: /api/1/graph/(graph_query)/ | .. http:get:: /api/1/graph/(graph_query)/ | ||||
| Show All 37 Lines | .. http:get:: /api/1/graph/(graph_query)/ | ||||
| :swh_web_api:`graph/visit/paths/swh:1:dir:644dd466d8ad527ea3a609bfd588a3244e6dafcb?direction=backward&resolve_origins=true` | :swh_web_api:`graph/visit/paths/swh:1:dir:644dd466d8ad527ea3a609bfd588a3244e6dafcb?direction=backward&resolve_origins=true` | ||||
| """ | """ | ||||
| return None | return None | ||||
| @api_route(r"/graph/(?P<graph_query>.+)/", "api-1-graph") | @api_route(r"/graph/(?P<graph_query>.+)/", "api-1-graph") | ||||
| @renderer_classes([PlainTextRenderer]) | @renderer_classes([PlainTextRenderer]) | ||||
| def api_graph_proxy(request: Request, graph_query: str) -> Response: | def api_graph_proxy( | ||||
| request: Request, graph_query: str | |||||
| ) -> Union[Response, StreamingHttpResponse]: | |||||
| if request.get_host() != SWH_WEB_INTERNAL_SERVER_NAME: | if request.get_host() != SWH_WEB_INTERNAL_SERVER_NAME: | ||||
| if not bool(request.user and request.user.is_authenticated): | if not bool(request.user and request.user.is_authenticated): | ||||
| return Response("Authentication credentials were not provided.", status=401) | return Response("Authentication credentials were not provided.", status=401) | ||||
| if not request.user.has_perm(API_GRAPH_PERM): | if not request.user.has_perm(API_GRAPH_PERM): | ||||
| return Response( | return Response( | ||||
| "You do not have permission to perform this action.", status=403 | "You do not have permission to perform this action.", status=403 | ||||
| ) | ) | ||||
| graph_query_url = get_config()["graph"]["server_url"] | graph_query_url = get_config()["graph"]["server_url"] | ||||
| graph_query_url += graph_query | graph_query_url += graph_query | ||||
| if request.GET: | if request.GET: | ||||
| graph_query_url += "?" + request.GET.urlencode(safe="/;:") | graph_query_url += "?" + request.GET.urlencode(safe="/;:") | ||||
| response = requests.get(graph_query_url) | response = requests.get(graph_query_url, stream=True) | ||||
| response_text = response.text | # graph stats and counter endpoint responses are not streamed | ||||
| if response.headers.get("Transfer-Encoding") != "chunked": | |||||
| return Response( | |||||
| response.text, | |||||
| status=response.status_code, | |||||
| content_type=response.headers["Content-Type"], | |||||
| ) | |||||
| # other endpoint responses are streamed | |||||
| else: | |||||
| resolve_origins = strtobool(request.GET.get("resolve_origins", "false")) | resolve_origins = strtobool(request.GET.get("resolve_origins", "false")) | ||||
| if response.status_code == 200 and resolve_origins: | if response.status_code == 200 and resolve_origins: | ||||
| response_text = _resolve_origin_swhids_in_graph_response(response) | response_stream = _resolve_origin_swhids_in_graph_response(response) | ||||
| return Response( | else: | ||||
| response_text, | response_stream = map(lambda line: line + b"\n", response.iter_lines()) | ||||
| return StreamingHttpResponse( | |||||
| response_stream, | |||||
| status=response.status_code, | status=response.status_code, | ||||
| content_type=response.headers["Content-Type"], | content_type=response.headers["Content-Type"], | ||||
| ) | ) | ||||