Differential D6007 Diff 21697 swh/web/common/utils.py

Changeset View

Standalone View

swh/web/common/utils.py

# Copyright (C) 2017-2021 The Software Heritage developers		# Copyright (C) 2017-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from datetime import datetime, timezone		from datetime import datetime, timezone
import os		import os
import re		import re
from typing import Any, Dict, Optional		from typing import Any, Dict, List, Optional

from bs4 import BeautifulSoup		from bs4 import BeautifulSoup
from docutils.core import publish_parts		from docutils.core import publish_parts
import docutils.parsers.rst		import docutils.parsers.rst
import docutils.utils		import docutils.utils
from docutils.writers.html5_polyglot import HTMLTranslator, Writer		from docutils.writers.html5_polyglot import HTMLTranslator, Writer
from iso8601 import ParseError, parse_date		from iso8601 import ParseError, parse_date
from pkg_resources import get_distribution		from pkg_resources import get_distribution
from prometheus_client.registry import CollectorRegistry		from prometheus_client.registry import CollectorRegistry
		import requests
		from requests.auth import HTTPBasicAuth

		from django.core.cache import cache
from django.http import HttpRequest, QueryDict		from django.http import HttpRequest, QueryDict
from django.urls import reverse as django_reverse		from django.urls import reverse as django_reverse

from swh.web.common.exc import BadInputExc		from swh.web.common.exc import BadInputExc
from swh.web.common.typing import QueryParameters		from swh.web.common.typing import QueryParameters
from swh.web.config import ORIGIN_VISIT_TYPES, get_config		from swh.web.config import ORIGIN_VISIT_TYPES, get_config

SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True)		SWH_WEB_METRICS_REGISTRY = CollectorRegistry(auto_describe=True)
▲ Show 20 Lines • Show All 321 Lines • ▼ Show 20 Lines	def prettify_html(html: str) -> str:

Args:		Args:
html: Input HTML document		html: Input HTML document

Returns:		Returns:
The prettified HTML document		The prettified HTML document
"""		"""
return BeautifulSoup(html, "lxml").prettify()		return BeautifulSoup(html, "lxml").prettify()


		def get_deposits_list() -> List[Dict[str, Any]]:
		"""Return the list of software deposits using swh-deposit API
		"""
		config = get_config()["deposit"]
		deposits_list_url = config["private_api_url"] + "deposits"
		deposits_list_auth = HTTPBasicAuth(
		config["private_api_user"], config["private_api_password"]
		)

		nb_deposits = requests.get(
		"%s?page_size=1" % deposits_list_url, auth=deposits_list_auth, timeout=30
		).json()["count"]

		deposits_data = cache.get("swh-deposit-list")
		if not deposits_data or deposits_data["count"] != nb_deposits:
		deposits_data = requests.get(
		"%s?page_size=%s" % (deposits_list_url, nb_deposits),
		auth=deposits_list_auth,
		timeout=30,
		).json()
		cache.set("swh-deposit-list", deposits_data)

		ardumontUnsubmitted Not Done Inline Actions .o0(food for thought) Seeing this code again, i can't help but think that we need to make that "private" [1] deposit api part a rpc server as we did for all the other modules we have... (as val already pointed out and maybe david before him...) [1] maybe all the deposit api? ardumont: .o0(food for thought) Seeing this code again, i can't help but think that we need to make that…
		anlambertAuthorUnsubmitted Done Inline Actions We could expose the private deposit API as an internal RPC service yes, this would remove the use of HTTP basic auth and requests module. Not a top priority at the moment but this will harmonize with other swh services for sure. anlambert: We could expose the private deposit API as an internal RPC service yes, this would remove the…
		return deposits_data["results"]