Changeset View
Changeset View
Standalone View
Standalone View
swh/web/misc/coverage.py
| Show All 10 Lines | |||||
| from django.http.request import HttpRequest | from django.http.request import HttpRequest | ||||
| from django.http.response import HttpResponse | from django.http.response import HttpResponse | ||||
| from django.shortcuts import render | from django.shortcuts import render | ||||
| from django.views.decorators.cache import never_cache | from django.views.decorators.cache import never_cache | ||||
| from django.views.decorators.clickjacking import xframe_options_exempt | from django.views.decorators.clickjacking import xframe_options_exempt | ||||
| from swh.scheduler.model import SchedulerMetrics | from swh.scheduler.model import SchedulerMetrics | ||||
| from swh.web.common import archive | from swh.web.common import archive | ||||
| from swh.web.common.origin_save import get_savable_visit_types | |||||
| from swh.web.common.utils import ( | from swh.web.common.utils import ( | ||||
| django_cache, | django_cache, | ||||
| get_deposits_list, | get_deposits_list, | ||||
| is_swh_web_development, | |||||
| is_swh_web_production, | is_swh_web_production, | ||||
| reverse, | reverse, | ||||
| ) | ) | ||||
| from swh.web.config import scheduler | from swh.web.config import scheduler | ||||
| _swh_arch_overview_doc = ( | _swh_arch_overview_doc = ( | ||||
| "https://docs.softwareheritage.org/devel/architecture/overview.html" | "https://docs.softwareheritage.org/devel/architecture/overview.html" | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 298 Lines • ▼ Show 20 Lines | def _search_url(query: str, visit_type: str) -> str: | ||||
| ) | ) | ||||
| @xframe_options_exempt | @xframe_options_exempt | ||||
| @never_cache | @never_cache | ||||
| def _swh_coverage(request: HttpRequest) -> HttpResponse: | def _swh_coverage(request: HttpRequest) -> HttpResponse: | ||||
| use_cache = is_swh_web_production(request) | use_cache = is_swh_web_production(request) | ||||
| listers_metrics = _get_listers_metrics(use_cache) | listers_metrics = _get_listers_metrics(use_cache) | ||||
| for origins in listed_origins["origins"]: | for origins in listed_origins["origins"]: | ||||
| origins["count"] = "0" | |||||
| origins["instances"] = {} | origins["instances"] = {} | ||||
| origins_type = origins["type"] | origins_type = origins["type"] | ||||
| # special processing for nixos/guix origins as there is no | # special processing for nixos/guix origins as there is no | ||||
| # scheduler metrics for those | # scheduler metrics for those | ||||
| if origins_type in ("nixos", "guix"): | if origins_type in ("nixos", "guix"): | ||||
| count = _get_nixguix_origins_count(origins["search_pattern"], use_cache) | count = _get_nixguix_origins_count(origins["search_pattern"], use_cache) | ||||
| origins["count"] = f"{count:,}" if count else "" | origins["count"] = f"{count:,}" | ||||
| origins["instances"][origins_type] = {"nixguix": {"count": count}} | origins["instances"][origins_type] = {"nixguix": {"count": count}} | ||||
| if origins_type not in listers_metrics: | if origins_type not in listers_metrics: | ||||
| continue | continue | ||||
| count_total = sum( | count_total = sum( | ||||
| [metrics.origins_known for _, metrics in listers_metrics[origins_type]] | [metrics.origins_enabled for _, metrics in listers_metrics[origins_type]] | ||||
| ) | ) | ||||
| count_never_visited = sum( | count_never_visited = sum( | ||||
| [ | [ | ||||
| metrics.origins_never_visited | metrics.origins_never_visited | ||||
| for _, metrics in listers_metrics[origins_type] | for _, metrics in listers_metrics[origins_type] | ||||
| ] | ] | ||||
| ) | ) | ||||
| count = count_total - count_never_visited | count = count_total - count_never_visited | ||||
| origins["count"] = f"{count:,}" | origins["count"] = f"{count:,}" | ||||
| origins["instances"] = defaultdict(dict) | origins["instances"] = defaultdict(dict) | ||||
| for instance, metrics in listers_metrics[origins_type]: | for instance, metrics in listers_metrics[origins_type]: | ||||
| # these types are available in staging/docker but not yet in production | instance_count = metrics.origins_enabled - metrics.origins_never_visited | ||||
| if ( | # no archived origins for that visit type, skip it | ||||
| metrics.visit_type in ("bzr", "cvs") | if instance_count == 0: | ||||
| and metrics.visit_type not in get_savable_visit_types() | |||||
| ): | |||||
| continue | continue | ||||
| instance_count = metrics.origins_known - metrics.origins_never_visited | |||||
| origins["instances"][instance].update( | origins["instances"][instance].update( | ||||
| {metrics.visit_type: {"count": f"{instance_count:,}"}} | {metrics.visit_type: {"count": f"{instance_count:,}"}} | ||||
| ) | ) | ||||
| origins["visit_types"] = list( | origins["visit_types"] = list( | ||||
| set(origins["instances"][instance].keys()) | set(origins["instances"][instance].keys()) | ||||
| | set(origins.get("visit_types", [])) | | set(origins.get("visit_types", [])) | ||||
| ) | ) | ||||
| Show All 10 Lines | for origins in listed_origins["origins"]: | ||||
| for visit_type in visit_types: | for visit_type in visit_types: | ||||
| if nb_instances > 1: | if nb_instances > 1: | ||||
| search_pattern = instance_name | search_pattern = instance_name | ||||
| else: | else: | ||||
| search_pattern = origins["search_pattern"] | search_pattern = origins["search_pattern"] | ||||
| search_url = _search_url(search_pattern, visit_type) | search_url = _search_url(search_pattern, visit_type) | ||||
| visit_types[visit_type]["search_url"] = search_url | visit_types[visit_type]["search_url"] = search_url | ||||
| # filter out origin types without archived origins on production and staging | |||||
| if not is_swh_web_development(request): | |||||
| listed_origins["origins"] = list( | |||||
| filter(lambda o: o["count"] != "0", listed_origins["origins"]) | |||||
| ) | |||||
| for origins in legacy_origins["origins"]: | for origins in legacy_origins["origins"]: | ||||
| origins["search_urls"] = {} | origins["search_urls"] = {} | ||||
| for visit_type in origins["visit_types"]: | for visit_type in origins["visit_types"]: | ||||
| origins["search_urls"][visit_type] = _search_url( | origins["search_urls"][visit_type] = _search_url( | ||||
| origins["search_pattern"], visit_type | origins["search_pattern"], visit_type | ||||
| ) | ) | ||||
| deposits_counts = _get_deposits_netloc_counts(use_cache) | deposits_counts = _get_deposits_netloc_counts(use_cache) | ||||
| for origins in deposited_origins["origins"]: | for origins in deposited_origins["origins"]: | ||||
| origins["count"] = "0" | |||||
| if origins["search_pattern"] in deposits_counts: | if origins["search_pattern"] in deposits_counts: | ||||
| origins["count"] = f"{deposits_counts[origins['search_pattern']]:,}" | origins["count"] = f"{deposits_counts[origins['search_pattern']]:,}" | ||||
| origins["search_urls"] = { | origins["search_urls"] = { | ||||
| "deposit": _search_url(origins["search_pattern"], "deposit") | "deposit": _search_url(origins["search_pattern"], "deposit") | ||||
| } | } | ||||
| focus = [] | focus = [] | ||||
| focus_param = request.GET.get("focus") | focus_param = request.GET.get("focus") | ||||
| Show All 20 Lines | |||||