Differential D7221 Diff 26178 swh/web/misc/coverage.py

Changeset View

Standalone View

swh/web/misc/coverage.py

# Copyright (C) 2018-2021 The Software Heritage developers		# Copyright (C) 2018-2022 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution		# See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version		# License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information		# See top-level LICENSE file for more information

from collections import Counter, defaultdict		from collections import Counter, defaultdict
from functools import lru_cache		from functools import lru_cache
from typing import Dict, List, Tuple		from typing import Dict, List, Tuple
from urllib.parse import urlparse		from urllib.parse import urlparse

import sentry_sdk		import sentry_sdk

from django.conf.urls import url		from django.conf.urls import url
from django.shortcuts import render		from django.shortcuts import render
from django.views.decorators.cache import never_cache		from django.views.decorators.cache import never_cache
from django.views.decorators.clickjacking import xframe_options_exempt		from django.views.decorators.clickjacking import xframe_options_exempt

from swh.scheduler.model import SchedulerMetrics		from swh.scheduler.model import SchedulerMetrics
from swh.web.common import archive		from swh.web.common import archive
		from swh.web.common.origin_save import get_savable_visit_types
from swh.web.common.utils import get_deposits_list, reverse		from swh.web.common.utils import get_deposits_list, reverse
from swh.web.config import scheduler		from swh.web.config import scheduler

_swh_arch_overview_doc = (		_swh_arch_overview_doc = (
"https://docs.softwareheritage.org/devel/architecture/overview.html"		"https://docs.softwareheritage.org/devel/architecture/overview.html"
)		)

# Current coverage list of the archive in a high level overview fashion,		# Current coverage list of the archive in a high level overview fashion,
▲ Show 20 Lines • Show All 297 Lines • ▼ Show 20 Lines	for origins in listed_origins["origins"]:
for _, metrics in listers_metrics[origins_type]		for _, metrics in listers_metrics[origins_type]
]		]
)		)
count = count_total - count_never_visited		count = count_total - count_never_visited

origins["count"] = f"{count:,}"		origins["count"] = f"{count:,}"
origins["instances"] = defaultdict(dict)		origins["instances"] = defaultdict(dict)
for instance, metrics in listers_metrics[origins_type]:		for instance, metrics in listers_metrics[origins_type]:
# not yet in production		# these types are available in staging/docker but not yet in production
if metrics.visit_type in ("bzr", "cvs"):		if (
		metrics.visit_type in ("bzr", "cvs")
		and metrics.visit_type not in get_savable_visit_types()
		):
continue		continue
instance_count = metrics.origins_known - metrics.origins_never_visited		instance_count = metrics.origins_known - metrics.origins_never_visited
origins["instances"][instance].update(		origins["instances"][instance].update(
{metrics.visit_type: {"count": f"{instance_count:,}"}}		{metrics.visit_type: {"count": f"{instance_count:,}"}}
)		)
origins["visit_types"] = list(		origins["visit_types"] = list(
set(origins["instances"][instance].keys())		set(origins["instances"][instance].keys())
\| set(origins.get("visit_types", []))		\| set(origins.get("visit_types", []))
▲ Show 20 Lines • Show All 51 Lines • Show Last 20 Lines