Page MenuHomeSoftware Heritage
Differential D6004 Diff 22361 swh/web/misc/coverage.py

Changeset View

Standalone View

View Options

swh/web/misc/coverage.py

# Copyright (C) 2018-2019 The Software Heritage developers # Copyright (C) 2018-2021 The Software Heritage developers
# See the AUTHORS file at the top-level directory of this distribution # See the AUTHORS file at the top-level directory of this distribution
# License: GNU Affero General Public License version 3, or any later version # License: GNU Affero General Public License version 3, or any later version
# See top-level LICENSE file for more information # See top-level LICENSE file for more information
from collections import Counter, defaultdict
from functools import lru_cache
from typing import Dict, List, Tuple
from urllib.parse import urlparse
import sentry_sdk
from django.conf.urls import url from django.conf.urls import url
from django.shortcuts import render from django.shortcuts import render
from django.views.decorators.clickjacking import xframe_options_exempt from django.views.decorators.clickjacking import xframe_options_exempt
from swh.web.config import get_config from swh.scheduler.model import SchedulerMetrics
from swh.web.common import archive
from swh.web.common.utils import get_deposits_list, reverse
from swh.web.config import scheduler
_swh_arch_overview_doc = (
"https://docs.softwareheritage.org/devel/architecture/overview.html"
)
# Current coverage list of the archive in a high level overview fashion,
# categorized as follow:
# - listed origins: origins discovered using a swh lister
# - legacy: origins where public hosting service has closed
# - deposited: origins coming from swh-deposit
#
ardumontUnsubmitted
Not Done
Inline Actions

or swh-web's?

ardumont: or swh-web's?
anlambertAuthorUnsubmitted
Done
Inline Actions

Those info could be needed in other swh components so I would not put that in swh-web db.

anlambert: Those info could be needed in other swh components so I would not put that in swh-web db.
ardumontUnsubmitted
Not Done
Inline Actions

sounds fair.

ardumont: sounds fair.
# TODO: Store that list in a database table somewhere (swh-scheduler, swh-storage ?)
# and retrieve it dynamically
listed_origins = {
"info": (
"These software origins get continuously discovered and archived using "
f'the <a href="{_swh_arch_overview_doc}#listers" target="_blank" '
'rel="noopener noreferrer">listers</a> implemented by Software Heritage.'
),
"origins": [
{
"type": "bitbucket",
"info_url": "https://bitbucket.org",
"info": "public repositories from Bitbucket",
"search_pattern": "https://bitbucket.org/",
},
{
"type": "cgit",
"info_url": "https://git.zx2c4.com/cgit/about",
"info": "public repositories from cgit instances",
"search_pattern": "cgit",
},
{
"type": "CRAN",
"info_url": "https://cran.r-project.org",
"info": "source packages from The Comprehensive R Archive Network",
"search_pattern": "https://cran.r-project.org/",
},
{
"type": "debian",
"info_url": "https://www.debian.org",
"info": "source packages from the Debian distribution",
"search_pattern": "deb://",
},
{
"type": "gitea",
"info_url": "https://gitea.io",
"info": "public repositories from Gitea instances",
"search_pattern": "gitea",
},
{
"type": "github",
"info_url": "https://github.com",
"info": "public repositories from GitHub",
"search_pattern": "https://github.com/",
},
{
"type": "gitlab",
"info_url": "https://gitlab.com",
"info": "public repositories from multiple GitLab instances",
"search_pattern": "gitlab",
},
{
"type": "guix",
"info_url": "https://guix.gnu.org",
"info": "source code tarballs used to build the Guix package collection",
ardumontUnsubmitted
Not Done
Inline Actions

should be like the other nixguix loader visit type is nixguix [1]
But those are complicated, they are dealing with other types like 'tar' for example.

That probably explains why we got so few in your screenshot (26) is too few to my taste ;)

[1] https://forge.softwareheritage.org/source/swh-loader-core/browse/master/swh/loader/package/nixguix/loader.py$68

ardumont: should be like the other nixguix loader visit type is nixguix [1] But those are complicated…
anlambertAuthorUnsubmitted
Done
Inline Actions

I missed the nixguix loader configuartion for guix, correct origin counts is the number of branches in that origin, will adapt to retrieve it.

anlambert: I missed the nixguix loader configuartion for guix, correct origin counts is the number of…
ardumontUnsubmitted
Not Done
Inline Actions

right!

ardumont: right!
"visit_types": ["nixguix"],
"search_pattern": "https://guix.gnu.org/sources.json",
},
{
"type": "GNU",
"info_url": "https://www.gnu.org",
"info": "releases from the GNU project (as of August 2015)",
"search_pattern": "gnu",
},
{
ardumontUnsubmitted
Not Done
Inline Actions
"info": (
- "The hosting services for these origins have been discontinued but hopefully "
- "Software Heritage managed to archive them."
+ "Discontinued hosting services. Those origins are now also archived "
+ "in Software Heritage."
),
"origins": [

(proposal)

ardumont: (proposal)
"type": "launchpad",
"info_url": "https://launchpad.net",
"logo": "img/logos/launchpad.png",
"info": "public repositories from Launchpad",
"search_pattern": "https://git.launchpad.net/",
},
{
ardumontUnsubmitted
Not Done
Inline Actions

i'd say nixpkgs here.

ardumont: i'd say nixpkgs here.
anlambertAuthorUnsubmitted
Done
Inline Actions

I use the type value to get associated png logo in static assets so I will keep nixos value here.

anlambert: I use the type value to get associated png logo in static assets so I will keep nixos value…
ardumontUnsubmitted
Not Done
Inline Actions

I use the type value to get associated png logo in static assets so I will keep nixos value here.

ack

ardumont: > I use the type value to get associated png logo in static assets so I will keep nixos value…
ardumontUnsubmitted
Not Done
Inline Actions

NixOS is the linux distribution (that uses nix and nix's dsl for everything).
Nixpkgs is the set of packages proposed by the nix package manager. That's what we are ingesting with the nixguix loader implementation (well a derivative of it through the json file just below).

I understood nixpkgs as the superset of all packages (including the one we can install for nixos).
Hence why i suggest we use nixpkgs here ;)

ardumont: NixOS is the linux distribution (that uses nix and nix's dsl for everything). Nixpkgs is the…
"type": "nixos",
"info_url": "https://nixos.org",
"info": "source code tarballs used to build the Nix package collection",
"visit_types": ["nixguix"],
"search_pattern": (
"https://nix-community.github.io/nixpkgs-swh/sources-unstable.json"
),
},
{
"type": "npm",
"info_url": "https://www.npmjs.com",
"info": "public packages from the package registry for javascript",
"search_pattern": "https://www.npmjs.com",
},
# apart our forge, most phabricator origins have not been archived
# while they have been listed so do not display those type of origins
# until new listing processes have been executed and origins loaded
#
# {
# "type": "phabricator",
# "info_url": "https://www.phacility.com/phabricator",
# "info": "public repositories from multiple Phabricator instances",
# "search_pattern": "phabricator",
# },
{
"type": "pypi",
"info_url": "https://pypi.org",
"info": "source packages from the Python Package Index",
"search_pattern": "https://pypi.org",
},
{
"type": "sourceforge",
"info_url": "https://sourceforge.net",
"info": "public repositories from SourceForge",
"search_pattern": "code.sf.net",
},
],
}
legacy_origins = {
"info": (
"Discontinued hosting services. Those origins have been archived "
"by Software Heritage."
),
"origins": [
{
"type": "gitorious",
"info_url": "https://en.wikipedia.org/wiki/Gitorious",
"info": (
"public repositories from the former Gitorious code hosting service"
),
"visit_types": ["git"],
"search_pattern": "https://gitorious.org",
"count": "122,014",
},
{
"type": "googlecode",
"info_url": "https://code.google.com/archive",
"info": (
"public repositories from the former Google Code project "
"hosting service"
),
"visit_types": ["git", "hg", "svn"],
"search_pattern": "googlecode.com",
"count": "790,026",
},
],
}
# Current coverage list of the archive deposited_origins = {
# TODO: Retrieve that list dynamically instead of hardcoding it "info": (
_code_providers = [ "These origins are directly pushed into the archive by trusted partners "
{ f'using the <a href="{_swh_arch_overview_doc}#deposit" target="_blank" '
"provider_id": "bitbucket", 'rel="noopener noreferrer">deposit</a> service of Software Heritage.'
"provider_url": "https://bitbucket.org/", ),
"provider_logo": "img/logos/bitbucket.png", "origins": [
"provider_info": "public repositories from Bitbucket " {
"(continuously archived)", "type": "elife",
"origin_url_regexp": "^https://bitbucket.org/", "info_url": "https://elifesciences.org",
"origin_types": "repositories", "info": (
}, "research software source code associated to the articles "
{ "eLife publishes"
"provider_id": "cran", ),
"provider_url": "https://cran.r-project.org/", "search_pattern": "elife.stencila.io",
"provider_logo": "img/logos/cran.svg", "visit_types": ["deposit"],
"provider_info": "source packages from The Comprehensive R Archive " },
"Network (continuously archived)", {
"origin_url_regexp": "^https://cran.r-project.org/", "type": "hal",
"origin_types": "packages", "info_url": "https://hal.archives-ouvertes.fr",
}, "info": "scientific software source code deposited in the open archive HAL",
{ "visit_types": ["deposit"],
"provider_id": "debian", "search_pattern": "hal.archives-ouvertes.fr",
"provider_url": "https://www.debian.org/", },
"provider_logo": "img/logos/debian.png", {
"provider_info": "source packages from the Debian distribution " "type": "ipol",
"(continuously archived)", "info_url": "https://www.ipol.im",
"origin_url_regexp": "^deb://", "info": "software artifacts associated to the articles IPOL publishes",
"origin_types": "packages", "visit_types": ["deposit"],
}, "search_pattern": "doi.org/10.5201",
{
"provider_id": "framagit",
"provider_url": "https://framagit.org/",
"provider_logo": "img/logos/framagit.png",
"provider_info": "public repositories from Framagit " "(continuously archived)",
"origin_url_regexp": "^https://framagit.org/",
"origin_types": "repositories",
},
{
"provider_id": "github",
"provider_url": "https://github.com",
"provider_logo": "img/logos/github.png",
"provider_info": "public repositories from GitHub " "(continuously archived)",
"origin_url_regexp": "^https://github.com/",
"origin_types": "repositories",
},
{
"provider_id": "gitlab",
"provider_url": "https://gitlab.com",
"provider_logo": "img/logos/gitlab.svg",
"provider_info": "public repositories from GitLab " "(continuously archived)",
"origin_url_regexp": "^https://gitlab.com/",
"origin_types": "repositories",
},
{
"provider_id": "gitorious",
"provider_url": "https://gitorious.org/",
"provider_logo": "img/logos/gitorious.png",
"provider_info": "public repositories from the former Gitorious code "
"hosting service",
"origin_url_regexp": "^https://gitorious.org/",
"origin_types": "repositories",
},
{
"provider_id": "googlecode",
"provider_url": "https://code.google.com/archive/",
"provider_logo": "img/logos/googlecode.png",
"provider_info": "public repositories from the former Google Code "
"project hosting service",
"origin_url_regexp": "^http.*.googlecode.com/",
"origin_types": "repositories",
},
{
"provider_id": "gnu",
"provider_url": "https://www.gnu.org",
"provider_logo": "img/logos/gnu.png",
"provider_info": "releases from the GNU project (as of August 2015)",
"origin_url_regexp": "^rsync://ftp.gnu.org/",
"origin_types": "releases",
},
{
"provider_id": "guix",
"provider_url": "https://guix.gnu.org/",
"provider_logo": "img/logos/guix.svg",
"provider_info": "source code tarballs used to build the Guix package "
"collection",
"origin_url_regexp": "^https://guix.gnu.org/",
"origin_types": "tarballs",
},
{
"provider_id": "hal",
"provider_url": "https://hal.archives-ouvertes.fr/",
"provider_logo": "img/logos/hal.png",
"provider_info": "scientific software source code deposited in the "
"open archive HAL",
"origin_url_regexp": "^https://hal.archives-ouvertes.fr/",
"origin_types": "deposits",
},
{
"provider_id": "inria",
"provider_url": "https://gitlab.inria.fr",
"provider_logo": "img/logos/inria.jpg",
"provider_info": "public repositories from Inria GitLab "
"(continuously archived)",
"origin_url_regexp": "^https://gitlab.inria.fr/",
"origin_types": "repositories",
},
{
"provider_id": "ipol",
"provider_url": "https://www.ipol.im/",
"provider_logo": "img/logos/ipol.png",
"provider_info": "software artifacts associated to the articles "
"IPOL publishes",
"origin_url_regexp": "^https://doi.org/10.5201/ipol",
"origin_types": "tarballs",
},
{
"provider_id": "npm",
"provider_url": "https://www.npmjs.com/",
"provider_logo": "img/logos/npm.png",
"provider_info": "public packages from the package registry for "
"javascript (continuously archived)",
"origin_url_regexp": "^https://www.npmjs.com/",
"origin_types": "packages",
},
{
"provider_id": "nixos",
"provider_url": "https://nixos.org/",
"provider_logo": "img/logos/nixos.png",
"provider_info": "source code tarballs used to build the Nix package "
"collection",
"origin_url_regexp": "^https://nix-community.github.io/nixpkgs-swh",
"origin_types": "tarballs",
},
{
"provider_id": "pypi",
"provider_url": "https://pypi.org",
"provider_logo": "img/logos/pypi.svg",
"provider_info": "source packages from the Python Packaging Index "
"(continuously archived)",
"origin_url_regexp": "^https://pypi.org/",
"origin_types": "packages",
}, },
],
}
vlorentzUnsubmitted
Done
Inline Actions

There should be a fallback when this fails (eg. because the scheduler isn't available, or not configured in the testing env)

vlorentz: There should be a fallback when this fails (eg. because the scheduler isn't available, or not…
anlambertAuthorUnsubmitted
Done
Inline Actions

I ensured that widget will still be displayed when metrics are not available, only counters info will be missing in that case.

anlambert: I ensured that widget will still be displayed when metrics are not available, only counters…
@lru_cache()
def _get_listers_metrics() -> Dict[str, List[Tuple[str, SchedulerMetrics]]]:
"""Returns scheduler metrics in the following mapping:
Dict[lister_name, List[Tuple[instance_name, SchedulerMetrics]]]
as a lister instance has one SchedulerMetrics object per visit type.
"""
listers_metrics = defaultdict(list)
try:
listers = scheduler().get_listers()
scheduler_metrics = scheduler().get_metrics()
for lister in listers:
for metrics in filter(
lambda m: m.lister_id == lister.id, scheduler_metrics
):
listers_metrics[lister.name].append((lister.instance_name, metrics))
except Exception as e:
sentry_sdk.capture_exception(e)
return listers_metrics
@lru_cache()
def _get_deposits_netloc_counts() -> Counter:
"""Return deposit counts per origin url network location.
"""
def _process_origin_url(origin_url):
parsed_url = urlparse(origin_url)
netloc = parsed_url.netloc
# special treatment for doi.org netloc as it is not specific enough
# for origins mapping
if parsed_url.netloc == "doi.org":
netloc += "/" + parsed_url.path.split("/")[1]
return netloc
netlocs = []
try:
deposits = get_deposits_list()
netlocs = [
_process_origin_url(d["origin_url"])
for d in deposits
if d["status"] == "done"
] ]
except Exception as e:
sentry_sdk.capture_exception(e)
return Counter(netlocs)
@lru_cache()
ardumontUnsubmitted
Not Done
Inline Actions
@lru_cache()
- def _get_nixos_origins_count(origin_url: str) -> int:
+ def _get_nixpkgs_origins_count(origin_url: str) -> int:
"""Returns number of archived tarballs for NixOS, aka the number
ardumont:
ardumontUnsubmitted
Not Done
Inline Actions
@lru_cache()
- def _get_nixos_origins_count(origin_url: str) -> int:
+ def _get_nixguix_origins_count(origin_url: str) -> int:
"""Returns number of archived tarballs for NixOS, aka the number

Scratch my previous suggestion change then ;)
Given one of your last comment, then this method should be named _nixguix_.
nixpkgs and guix origins counts should be computed as this function does (counting the number of branches in the snapshots).

ardumont: Scratch my previous suggestion change then ;) Given one of your last comment, then this method…
anlambertAuthorUnsubmitted
Done
Inline Actions

already done ;-) update incoming

anlambert: already done ;-) update incoming
ardumontUnsubmitted
Not Done
Inline Actions

*thumbs up*

ardumont: *thumbs up*
def _get_nixguix_origins_count(origin_url: str) -> int:
"""Returns number of archived tarballs for NixOS, aka the number
of branches in a dedicated origin in the archive.
"""
snapshot = archive.lookup_latest_origin_snapshot(origin_url)
if snapshot:
snapshot_sizes = archive.lookup_snapshot_sizes(snapshot["id"])
return snapshot_sizes["revision"]
else:
return 0
def _search_url(query: str, visit_type: str) -> str:
return reverse(
"browse-search",
query_params={
"q": query,
"visit_type": visit_type,
"with_visit": "true",
"with_content": "true",
},
)
@xframe_options_exempt @xframe_options_exempt
ardumontUnsubmitted
Not Done
Inline Actions

til [1]

[1] https://docs.djangoproject.com/en/3.2/ref/clickjacking/#setting-x-frame-options-for-all-responses

ardumont: til [1] [1] https://docs.djangoproject.com/en/3.2/ref/clickjacking/#setting-x-frame-options…
anlambertAuthorUnsubmitted
Done
Inline Actions

Originally that view was also integrated in an iframe on softwareheritage.org but this is no longer the case.
I kept that particular view implementation in case we want to integrate it again.

anlambert: Originally that view was also integrated in an iframe on softwareheritage.org but this is no…
def _swh_coverage(request): def _swh_coverage(request):
count_origins = get_config()["coverage_count_origins"] listers_metrics = _get_listers_metrics()
for origins in listed_origins["origins"]:
origins["instances"] = {}
origins_type = origins["type"]
# special processing for nixos/guix origins as there is no
# scheduler metrics for those
if origins_type in ("nixos", "guix"):
count = _get_nixguix_origins_count(origins["search_pattern"])
ardumontUnsubmitted
Not Done
Inline Actions

how weird!?

ardumont: how weird!?
anlambertAuthorUnsubmitted
Done
Inline Actions

Yes, do not know what is wrong here but this what we currently have in scheduler_metrics table:

softwareheritage-scheduler=> select name, instance_name, last_update, origins_known, origins_enabled, origins_never_visited, origins_with_pending_changes from listers inner join scheduler_metrics on id = lister_id order by name;
     name      |        instance_name         |          last_update          | origins_known | origins_enabled | origins_never_visited | origins_with_pending_changes 
---------------+------------------------------+-------------------------------+---------------+-----------------+-----------------------+------------------------------
 CRAN          | cran                         | 2021-07-13 12:14:07.919027+00 |         18292 |           18292 |                 18292 |                            0
anlambert: Yes, do not know what is wrong here but this what we currently have in scheduler_metrics table…
origins["count"] = count
origins["instances"][origins_type] = {"nixguix": {"count": count}}
if origins_type not in listers_metrics:
continue
count = sum(
[metrics.origins_known for _, metrics in listers_metrics[origins_type]]
)
count_never_visited = sum(
ardumontUnsubmitted
Not Done
Inline Actions
"info": "software artifacts associated to the articles IPOL publishes",
- "netloc": "doi.org",
+ "netloc": "doi.org/10.5201",
},
]

That might pose an issue because long term, other deposit clients could also reference their origin through the doi site...
I don't know how deep that breaks the current assumpion on the netloc...

The ipol specific doi is https://doi.org/10.5201/ so might be change the netloc entry to what the changes suggest?

Or maybe it's yagni, let's consider this when the time comes?

At least now you know ^ ;)

ardumont: That might pose an issue because long term, other deposit clients could also reference their…
anlambertAuthorUnsubmitted
Done
Inline Actions

I think the best way is to add a special processing for doi.org netloc and add the first path to it.

anlambert: I think the best way is to add a special processing for doi.org netloc and add the first path…
[
metrics.origins_never_visited
for _, metrics in listers_metrics[origins_type]
]
)
# CRAN origins are currently marked as not visited while they have been
if origins_type != "CRAN":
count -= count_never_visited
origins["count"] = f"{count:,}"
origins["instances"] = defaultdict(dict)
for instance, metrics in listers_metrics[origins_type]:
# not yet in production
if metrics.visit_type in ("bzr", "cvs"):
continue
origins["instances"][instance].update(
{
metrics.visit_type: {
"count": metrics.origins_known - metrics.origins_never_visited
vlorentzUnsubmitted
Not Done
Inline Actions

crashes with KeyError: 'instances' in an empty docker instance.

vlorentz: crashes with `KeyError: 'instances'` in an empty docker instance.
}
}
)
origins["visit_types"] = list(
set(origins["instances"][instance].keys())
| set(origins.get("visit_types", []))
)
if origins_type == "CRAN":
origins["instances"]["cran"]["cran"] = {"count": origins["count"]}
# defaultdict cannot be iterated in django template
origins["instances"] = dict(origins["instances"])
for origins in listed_origins["origins"]:
instances = origins["instances"]
nb_instances = len(instances)
for instance_name, visit_types in instances.items():
for visit_type in visit_types:
if nb_instances > 1:
search_pattern = instance_name
else:
search_pattern = origins["search_pattern"]
search_url = _search_url(search_pattern, visit_type)
visit_types[visit_type]["search_url"] = search_url
for origins in legacy_origins["origins"]:
origins["search_urls"] = {}
for visit_type in origins["visit_types"]:
origins["search_urls"][visit_type] = _search_url(
origins["search_pattern"], visit_type
)
deposits_counts = _get_deposits_netloc_counts()
for origins in deposited_origins["origins"]:
if origins["search_pattern"] in deposits_counts:
origins["count"] = f"{deposits_counts[origins['search_pattern']]:,}"
origins["search_urls"] = {
"deposit": _search_url(origins["search_pattern"], "deposit")
}
return render( return render(
request, request,
"misc/coverage.html", "misc/coverage.html",
{"providers": _code_providers, "count_origins": count_origins}, {
"origins": {
"Regular crawling": listed_origins,
"Discontinued hosting": legacy_origins,
"On demand archival": deposited_origins,
}
},
) )
urlpatterns = [ urlpatterns = [
url(r"^coverage/$", _swh_coverage, name="swh-coverage"), url(r"^coverage/$", _swh_coverage, name="swh-coverage"),
] ]
About · Developer information · Legal