diff --git a/swh/web/browse/snapshot_context.py b/swh/web/browse/snapshot_context.py --- a/swh/web/browse/snapshot_context.py +++ b/swh/web/browse/snapshot_context.py @@ -485,7 +485,11 @@ releases = list(reversed(releases)) - snapshot_sizes = archive.lookup_snapshot_sizes(snapshot_id) + snapshot_sizes_cache_id = f"swh_snapshot_{snapshot_id}_sizes" + snapshot_sizes = cache.get(snapshot_sizes_cache_id) + if snapshot_sizes is None: + snapshot_sizes = archive.lookup_snapshot_sizes(snapshot_id) + cache.set(snapshot_sizes_cache_id, snapshot_sizes) is_empty = sum(snapshot_sizes.values()) == 0 diff --git a/swh/web/common/archive.py b/swh/web/common/archive.py --- a/swh/web/common/archive.py +++ b/swh/web/common/archive.py @@ -985,7 +985,7 @@ return converters.from_origin_visit({**visit_status.to_dict(), "type": visit.type}) -def lookup_snapshot_sizes(snapshot_id): +def lookup_snapshot_sizes(snapshot_id: str) -> Dict[str, int]: """Count the number of branches in the snapshot with the given id Args: @@ -1003,23 +1003,49 @@ snapshot_sizes["release"] = 0 # adjust revision / release count for display if aliases are defined if "alias" in snapshot_sizes: - aliases = lookup_snapshot( - snapshot_id, branches_count=snapshot_sizes["alias"], target_types=["alias"] - ) - for alias in aliases["branches"].values(): - try: - for target_type in ("revision", "release"): - snapshot = lookup_snapshot( + # iterate on all snapshot branches to resolve aliases but stop once + # all aliases have been processed + batch_size = 10000 + snapshot = lookup_snapshot(snapshot_id, branches_count=batch_size) + nb_processed_aliases = 0 + while snapshot is not None: + for branch_name, branch_info in snapshot["branches"].items(): + if branch_info["target_type"] == "alias": + nb_processed_aliases += 1 + branches = lookup_snapshot( snapshot_id, - branches_from=alias["target"], + branches_from=branch_info["target"], branches_count=1, - target_types=[target_type], - ) - if snapshot and alias["target"] in snapshot["branches"]: - snapshot_sizes[target_type] += 1 - except NotFoundExc: - # aliased revision or release is missing in the snapshot - pass + )["branches"] + if ( + branch_info["target"] in branches + and branches[branch_info["target"]] is not None + ): + target = branches[branch_info["target"]]["target"] + for lookup_fn, target_type in ( + (lookup_revision, "revision"), + (lookup_release, "release"), + ): + try: + lookup_fn(target) + snapshot_sizes[target_type] += 1 + break + except NotFoundExc: + # aliased revision or release is missing in the snapshot + pass + if nb_processed_aliases == snapshot_sizes["alias"]: + break + if nb_processed_aliases == snapshot_sizes["alias"]: + break + if snapshot["next_branch"] is not None: + snapshot = lookup_snapshot( + snapshot_id, + branches_from=snapshot["next_branch"], + branches_count=batch_size, + ) + else: + snapshot = None + del snapshot_sizes["alias"] # remove possible None key returned by snapshot_count_branches # when null branches are present in the snapshot