diff --git a/swh/web/browse/snapshot_context.py b/swh/web/browse/snapshot_context.py --- a/swh/web/browse/snapshot_context.py +++ b/swh/web/browse/snapshot_context.py @@ -6,7 +6,6 @@ # Utility module for browsing the archive in a snapshot context. from collections import defaultdict -from copy import copy from typing import Any, Dict, List, Optional, Tuple from django.core.cache import cache @@ -203,6 +202,7 @@ if target_type == "revision": branches[branch_name] = SnapshotBranchInfo( name=branch_name, + alias=False, revision=target_id, date=None, directory=None, @@ -216,9 +216,10 @@ branch_aliases[branch_name] = target_id # FIXME: handle pointers to other object types - def _add_release_info(branch, release): + def _add_release_info(branch, release, alias=False): releases[branch] = SnapshotReleaseInfo( name=release["name"], + alias=alias, branch_name=branch, date=format_utc_iso_date(release["date"]), directory=None, @@ -229,9 +230,10 @@ url=None, ) - def _add_branch_info(branch, revision): + def _add_branch_info(branch, revision, alias=False): branches[branch] = SnapshotBranchInfo( name=branch, + alias=alias, revision=revision["id"], directory=revision["directory"], date=format_utc_iso_date(revision["date"]), @@ -262,26 +264,20 @@ releases[release_id]["directory"] = revision["directory"] for branch_alias, branch_target in branch_aliases.items(): - if branch_target in branches: - branches[branch_alias] = copy(branches[branch_target]) - else: - snp = archive.lookup_snapshot( - snapshot["id"], branches_from=branch_target, branches_count=1 - ) - if snp and branch_target in snp["branches"]: + resolved_alias = archive.lookup_snapshot_alias(snapshot["id"], branch_alias) + if resolved_alias is None: + continue - if snp["branches"][branch_target] is None: - continue + target_type = resolved_alias["target_type"] + target = resolved_alias["target"] - target_type = snp["branches"][branch_target]["target_type"] - target = snp["branches"][branch_target]["target"] - if target_type == "revision": - branches[branch_alias] = snp["branches"][branch_target] - revision = archive.lookup_revision(target) - _add_branch_info(branch_alias, revision) - elif target_type == "release": - release = archive.lookup_release(target) - _add_release_info(branch_alias, release) + if target_type == "revision": + branches[branch_alias] = target + revision = archive.lookup_revision(target) + _add_branch_info(branch_alias, revision, alias=True) + elif target_type == "release": + release = archive.lookup_release(target) + _add_release_info(branch_alias, release, alias=True) if branch_alias in branches: branches[branch_alias]["name"] = branch_alias @@ -487,7 +483,7 @@ snapshot_sizes = archive.lookup_snapshot_sizes(snapshot_id) - is_empty = sum(snapshot_sizes.values()) == 0 + is_empty = (snapshot_sizes["release"] + snapshot_sizes["revision"]) == 0 swh_snp_id = swhid("snapshot", snapshot_id) @@ -502,7 +498,7 @@ release_id = None root_directory = None - snapshot_total_size = sum(snapshot_sizes.values()) + snapshot_total_size = snapshot_sizes["release"] + snapshot_sizes["revision"] if path is not None: query_params["path"] = path @@ -513,6 +509,7 @@ branches.append( SnapshotBranchInfo( name=revision_id, + alias=False, revision=revision_id, directory=root_directory, date=revision["date"], diff --git a/swh/web/common/archive.py b/swh/web/common/archive.py --- a/swh/web/common/archive.py +++ b/swh/web/common/archive.py @@ -14,7 +14,7 @@ from swh.model.model import OriginVisit, Revision from swh.storage.algos import diff, revisions_walker from swh.storage.algos.origin import origin_get_latest_visit_status -from swh.storage.algos.snapshot import snapshot_get_latest +from swh.storage.algos.snapshot import snapshot_get_latest, snapshot_resolve_alias from swh.vault.exc import NotFoundExc as VaultNotFoundExc from swh.web import config from swh.web.common import converters, query @@ -993,7 +993,7 @@ return converters.from_origin_visit({**visit_status.to_dict(), "type": visit.type}) -def lookup_snapshot_sizes(snapshot_id): +def lookup_snapshot_sizes(snapshot_id: str) -> Dict[str, int]: """Count the number of branches in the snapshot with the given id Args: @@ -1004,34 +1004,12 @@ values their corresponding amount """ snapshot_id_bin = _to_sha1_bin(snapshot_id) - snapshot_sizes = storage.snapshot_count_branches(snapshot_id_bin) - if "revision" not in snapshot_sizes: - snapshot_sizes["revision"] = 0 - if "release" not in snapshot_sizes: - snapshot_sizes["release"] = 0 - # adjust revision / release count for display if aliases are defined - if "alias" in snapshot_sizes: - aliases = lookup_snapshot( - snapshot_id, branches_count=snapshot_sizes["alias"], target_types=["alias"] - ) - for alias in aliases["branches"].values(): - try: - for target_type in ("revision", "release"): - snapshot = lookup_snapshot( - snapshot_id, - branches_from=alias["target"], - branches_count=1, - target_types=[target_type], - ) - if snapshot and alias["target"] in snapshot["branches"]: - snapshot_sizes[target_type] += 1 - except NotFoundExc: - # aliased revision or release is missing in the snapshot - pass - del snapshot_sizes["alias"] - # remove possible None key returned by snapshot_count_branches - # when null branches are present in the snapshot - snapshot_sizes.pop(None, None) + snapshot_sizes = dict.fromkeys(("alias", "release", "revision"), 0) + branch_counts = storage.snapshot_count_branches(snapshot_id_bin) + # remove possible None key returned by snapshot_count_branches + # when null branches are present in the snapshot + branch_counts.pop(None, None) + snapshot_sizes.update(branch_counts) return snapshot_sizes @@ -1123,6 +1101,29 @@ return None +def lookup_snapshot_alias( + snapshot_id: str, alias_name: str +) -> Optional[Dict[str, Any]]: + """Try to resolve a branch alias in a snapshot. + + Args: + snapshot_id: hexadecimal representation of a snapshot id + alias_name: name of the branch alias to resolve + + Returns: + Target branch information or None if the alias does not exist + or target a dangling branch. + """ + resolved_alias = snapshot_resolve_alias( + storage, _to_sha1_bin(snapshot_id), alias_name.encode() + ) + return ( + converters.from_swh(resolved_alias.to_dict(), hashess={"target"}) + if resolved_alias is not None + else None + ) + + def lookup_revision_through(revision, limit=100): """Retrieve a revision from the criterion stored in revision dictionary. diff --git a/swh/web/common/typing.py b/swh/web/common/typing.py --- a/swh/web/common/typing.py +++ b/swh/web/common/typing.py @@ -56,6 +56,8 @@ """message of branch heading revision""" name: str """branch name""" + alias: bool + """define if the branch is an alias""" revision: str """branch heading revision""" url: Optional[str] @@ -75,6 +77,8 @@ """release message""" name: str """release name""" + alias: bool + """define if the branch is an alias""" target: str """release target""" target_type: str diff --git a/swh/web/tests/browse/test_snapshot_context.py b/swh/web/tests/browse/test_snapshot_context.py --- a/swh/web/tests/browse/test_snapshot_context.py +++ b/swh/web/tests/browse/test_snapshot_context.py @@ -39,12 +39,13 @@ branches = [] releases = [] - def _process_branch_data(branch, branch_data): + def _process_branch_data(branch, branch_data, alias=False): if branch_data["target_type"] == "revision": rev_data = archive_data.revision_get(branch_data["target"]) branches.append( SnapshotBranchInfo( name=branch, + alias=alias, revision=branch_data["target"], directory=rev_data["directory"], date=format_utc_iso_date(rev_data["date"]), @@ -58,6 +59,7 @@ releases.append( SnapshotReleaseInfo( name=rel_data["name"], + alias=alias, branch_name=branch, date=format_utc_iso_date(rel_data["date"]), id=rel_data["id"], @@ -73,7 +75,7 @@ branch_data = snapshot["branches"][branch] if branch_data["target_type"] == "alias": target_data = snapshot["branches"][branch_data["target"]] - _process_branch_data(branch, target_data) + _process_branch_data(branch, target_data, alias=True) else: _process_branch_data(branch, branch_data) @@ -126,7 +128,7 @@ releases_url = reverse("browse-snapshot-releases", url_args=url_args) is_empty = not branches and not releases snapshot_swhid = gen_swhid("snapshot", snapshot) - snapshot_sizes = {"revision": len(branches), "release": len(releases)} + snapshot_sizes = archive_data.snapshot_count_branches(snapshot) expected = SnapshotContext( branch="HEAD", @@ -217,7 +219,7 @@ ) is_empty = not branches and not releases snapshot_swhid = gen_swhid("snapshot", snapshot) - snapshot_sizes = {"revision": len(branches), "release": len(releases)} + snapshot_sizes = archive_data.snapshot_count_branches(snapshot) visit_info["url"] = reverse( "browse-origin-directory", query_params=query_params @@ -341,6 +343,7 @@ branches.append( SnapshotBranchInfo( name=revision["id"], + alias=False, revision=revision["id"], directory=revision["directory"], date=revision["date"], diff --git a/swh/web/tests/browse/views/test_content.py b/swh/web/tests/browse/views/test_content.py --- a/swh/web/tests/browse/views/test_content.py +++ b/swh/web/tests/browse/views/test_content.py @@ -409,6 +409,7 @@ visits = archive_data.origin_visit_get(origin["url"]) visit = random.choice(visits) snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(visit["snapshot"]) branches, releases = process_snapshot_branches(snapshot) branch_info = random.choice(branches) @@ -433,7 +434,9 @@ client, url, status_code=200, template_used="browse/content.html" ) - _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases) + _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases + ) assert_contains(resp, directory_file["name"]) assert_contains(resp, f"Branch: {branch_info['name']}") @@ -476,6 +479,7 @@ visits = archive_data.origin_visit_get(origin["url"]) visit = random.choice(visits) snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(visit["snapshot"]) branches, releases = process_snapshot_branches(snapshot) release_info = random.choice(releases) @@ -499,7 +503,9 @@ client, url, status_code=200, template_used="browse/content.html" ) - _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases) + _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases + ) assert_contains(resp, directory_file["name"]) assert_contains(resp, f"Release: {release_info['name']}") @@ -544,7 +550,9 @@ assert_contains(resp, snp_swhid) -def _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases): +def _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases +): browse_origin_url = reverse( "browse-origin", query_params={"origin_url": origin["url"]} ) @@ -556,7 +564,7 @@ ) assert_contains(resp, f'href="{escape(origin_branches_url)}"') - assert_contains(resp, f"Branches ({len(branches)})") + assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse( "browse-origin-releases", @@ -564,7 +572,7 @@ ) assert_contains(resp, f'href="{escape(origin_releases_url)}"') - assert_contains(resp, f"Releases ({len(releases)})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']})") assert_contains(resp, '
  • ', count=len(branches)) assert_contains(resp, '
  • ', count=len(releases)) diff --git a/swh/web/tests/browse/views/test_directory.py b/swh/web/tests/browse/views/test_directory.py --- a/swh/web/tests/browse/views/test_directory.py +++ b/swh/web/tests/browse/views/test_directory.py @@ -186,6 +186,7 @@ visits = archive_data.origin_visit_get(origin["url"]) visit = random.choice(visits) snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(visit["snapshot"]) branches, releases = process_snapshot_branches(snapshot) branch_info = random.choice(branches) @@ -210,7 +211,9 @@ client, url, status_code=200, template_used="browse/directory.html" ) - _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases) + _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases + ) assert_contains(resp, directory_subdir["name"]) assert_contains(resp, f"Branch: {branch_info['name']}") @@ -242,6 +245,7 @@ visits = archive_data.origin_visit_get(origin["url"]) visit = random.choice(visits) snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(visit["snapshot"]) branches, releases = process_snapshot_branches(snapshot) release_info = random.choice(releases) @@ -266,7 +270,9 @@ client, url, status_code=200, template_used="browse/directory.html" ) - _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases) + _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases + ) assert_contains(resp, directory_subdir["name"]) assert_contains(resp, f"Release: {release_info['name']}") @@ -300,7 +306,9 @@ assert_contains(resp, snp_swhid) -def _check_origin_snapshot_related_html(resp, origin, snapshot, branches, releases): +def _check_origin_snapshot_related_html( + resp, origin, snapshot, snapshot_sizes, branches, releases +): browse_origin_url = reverse( "browse-origin", query_params={"origin_url": origin["url"]} ) @@ -313,7 +321,7 @@ ) assert_contains(resp, f'href="{escape(origin_branches_url)}"') - assert_contains(resp, f"Branches ({len(branches)})") + assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse( "browse-origin-releases", @@ -321,7 +329,7 @@ ) assert_contains(resp, f'href="{escape(origin_releases_url)}"') - assert_contains(resp, f"Releases ({len(releases)})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']})") assert_contains(resp, '
  • ', count=len(branches)) assert_contains(resp, '
  • ', count=len(releases)) diff --git a/swh/web/tests/browse/views/test_origin.py b/swh/web/tests/browse/views/test_origin.py --- a/swh/web/tests/browse/views/test_origin.py +++ b/swh/web/tests/browse/views/test_origin.py @@ -83,6 +83,7 @@ "root_dir_sha1": head_rev["directory"], "content": get_content(dir_file["checksums"]["sha1"]), "visit": origin_visits[visit_idx], + "snapshot_sizes": archive_data.snapshot_count_branches(snapshot["id"]), } tdata = _get_archive_data(-1) @@ -92,6 +93,7 @@ archive_data, origin, origin_visits[-1], + tdata["snapshot_sizes"], tdata["branches"], tdata["releases"], tdata["root_dir_sha1"], @@ -103,6 +105,7 @@ archive_data, origin, origin_visits[-1], + tdata["snapshot_sizes"], tdata["branches"], tdata["releases"], tdata["root_dir_sha1"], @@ -115,6 +118,7 @@ archive_data, origin, origin_visits[-1], + tdata["snapshot_sizes"], tdata["branches"], tdata["releases"], tdata["root_dir_sha1"], @@ -129,6 +133,7 @@ archive_data, origin, origin_visits[0], + tdata["snapshot_sizes"], tdata["branches"], tdata["releases"], tdata["root_dir_sha1"], @@ -141,6 +146,7 @@ archive_data, origin, origin_visits[0], + tdata["snapshot_sizes"], tdata["branches"], tdata["releases"], tdata["root_dir_sha1"], @@ -155,6 +161,7 @@ visit = origin_visits[-1] snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"]) head_rev_id = archive_data.snapshot_get_head(snapshot) head_rev = archive_data.revision_get(head_rev_id) root_dir_sha1 = head_rev["directory"] @@ -166,6 +173,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -177,6 +185,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -189,6 +198,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -201,6 +211,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -213,6 +224,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -224,6 +236,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -236,6 +249,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -248,6 +262,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -262,6 +277,7 @@ visit = origin_visits[-1] snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"]) head_rev_id = archive_data.snapshot_get_head(snapshot) head_rev = archive_data.revision_get(head_rev_id) root_dir_sha1 = head_rev["directory"] @@ -282,6 +298,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -294,6 +311,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -307,6 +325,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -320,6 +339,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -333,6 +353,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -345,6 +366,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -358,6 +380,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -371,6 +394,7 @@ archive_data, origin, visit, + snapshot_sizes, branches, releases, root_dir_sha1, @@ -386,12 +410,13 @@ visit = origin_visits[-1] snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"]) snapshot_content = process_snapshot_branches(snapshot) - _origin_branches_test_helper(client, origin, snapshot_content) + _origin_branches_test_helper(client, origin, snapshot_content, snapshot_sizes) _origin_branches_test_helper( - client, origin, snapshot_content, snapshot_id=visit["snapshot"] + client, origin, snapshot_content, snapshot_sizes, snapshot_id=visit["snapshot"] ) @@ -401,12 +426,13 @@ visit = origin_visits[-1] snapshot = archive_data.snapshot_get(visit["snapshot"]) + snapshot_sizes = archive_data.snapshot_count_branches(snapshot["id"]) snapshot_content = process_snapshot_branches(snapshot) - _origin_releases_test_helper(client, origin, snapshot_content) + _origin_releases_test_helper(client, origin, snapshot_content, snapshot_sizes) _origin_releases_test_helper( - client, origin, snapshot_content, snapshot_id=visit["snapshot"] + client, origin, snapshot_content, snapshot_sizes, snapshot_id=visit["snapshot"] ) @@ -831,6 +857,7 @@ archive_data, origin_info, origin_visit, + snapshot_sizes, origin_branches, origin_releases, root_dir_sha1, @@ -907,12 +934,12 @@ origin_branches_url = reverse("browse-origin-branches", query_params=query_params) assert_contains(resp, f'href="{escape(origin_branches_url)}"') - assert_contains(resp, f"Branches ({len(origin_branches)})") + assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse("browse-origin-releases", query_params=query_params) assert_contains(resp, f'href="{escape(origin_releases_url)}">') - assert_contains(resp, f"Releases ({len(origin_releases)})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']})") assert_contains(resp, '
  • ', count=len(origin_branches)) @@ -968,6 +995,7 @@ archive_data, origin_info, origin_visit, + snapshot_sizes, origin_branches, origin_releases, root_directory_sha1, @@ -1044,14 +1072,14 @@ origin_branches_url = reverse("browse-origin-branches", query_params=query_params) assert_contains(resp, f'href="{escape(origin_branches_url)}"') - assert_contains(resp, f"Branches ({len(origin_branches)})") + assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse("browse-origin-releases", query_params=query_params) nb_releases = len(origin_releases) if nb_releases > 0: assert_contains(resp, f'href="{escape(origin_releases_url)}"') - assert_contains(resp, f"Releases ({nb_releases})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']})") if path: query_params["path"] = path @@ -1103,7 +1131,7 @@ def _origin_branches_test_helper( - client, origin_info, origin_snapshot, snapshot_id=None + client, origin_info, origin_snapshot, snapshot_sizes, snapshot_id=None ): query_params = {"origin_url": origin_info["url"], "snapshot": snapshot_id} @@ -1119,14 +1147,14 @@ origin_branches_url = reverse("browse-origin-branches", query_params=query_params) assert_contains(resp, f'href="{escape(origin_branches_url)}"') - assert_contains(resp, f"Branches ({len(origin_branches)})") + assert_contains(resp, f"Branches ({snapshot_sizes['revision']})") origin_releases_url = reverse("browse-origin-releases", query_params=query_params) nb_releases = len(origin_releases) if nb_releases > 0: assert_contains(resp, f'href="{escape(origin_releases_url)}">') - assert_contains(resp, f"Releases ({nb_releases})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']})") assert_contains(resp, ' 0: assert_contains(resp, f'href="{escape(origin_releases_url)}"') - assert_contains(resp, f"Releases ({nb_releases})") + assert_contains(resp, f"Releases ({snapshot_sizes['release']}") assert_contains(resp, '