Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/loader.py
Show First 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | ): | ||||
self.incremental = incremental | self.incremental = incremental | ||||
self.statsd = statsd | self.statsd = statsd | ||||
if base_snapshots and incremental: | if base_snapshots and incremental: | ||||
self.base_snapshots: List[Snapshot] = base_snapshots | self.base_snapshots: List[Snapshot] = base_snapshots | ||||
else: | else: | ||||
self.base_snapshots = [] | self.base_snapshots = [] | ||||
self.heads: Set[HexBytes] = set() | # Cache existing heads | ||||
self.local_heads: Set[HexBytes] = set() | |||||
def get_parents(self, commit: bytes) -> List[bytes]: | heads_logger.debug("Heads known in the archive:") | ||||
"""This method should return the list of known parents""" | for base_snapshot in self.base_snapshots: | ||||
return [] | for branch_name, branch in base_snapshot.branches.items(): | ||||
if not branch or branch.target_type == TargetType.ALIAS: | |||||
continue | |||||
heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | |||||
self.local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | |||||
def graph_walker(self) -> ObjectStoreGraphWalker: | def graph_walker(self) -> ObjectStoreGraphWalker: | ||||
return ObjectStoreGraphWalker(self.heads, self.get_parents) | return ObjectStoreGraphWalker(self.local_heads, get_parents=lambda commit: []) | ||||
def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: | def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: | ||||
"""Get the list of bytehex sha1s that the git loader should fetch. | """Get the list of bytehex sha1s that the git loader should fetch. | ||||
This compares the remote refs sent by the server with the base snapshot | This compares the remote refs sent by the server with the base snapshot | ||||
provided by the loader. | provided by the loader. | ||||
""" | """ | ||||
if not refs: | if not refs: | ||||
return [] | return [] | ||||
if heads_logger.isEnabledFor(logging.DEBUG): | if heads_logger.isEnabledFor(logging.DEBUG): | ||||
heads_logger.debug("Heads returned by the git remote:") | heads_logger.debug("Heads returned by the git remote:") | ||||
for name, value in refs.items(): | for name, value in refs.items(): | ||||
heads_logger.debug(" %r: %s", name, value.decode()) | heads_logger.debug(" %r: %s", name, value.decode()) | ||||
heads_logger.debug("Heads known in the archive:") | |||||
# Cache existing heads | |||||
local_heads: Set[HexBytes] = set() | |||||
for base_snapshot in self.base_snapshots: | |||||
for branch_name, branch in base_snapshot.branches.items(): | |||||
if not branch or branch.target_type == TargetType.ALIAS: | |||||
continue | |||||
heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | |||||
local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | |||||
self.heads = local_heads | |||||
# Get the remote heads that we want to fetch | # Get the remote heads that we want to fetch | ||||
remote_heads: Set[HexBytes] = set() | remote_heads: Set[HexBytes] = set() | ||||
for ref_name, ref_target in refs.items(): | for ref_name, ref_target in refs.items(): | ||||
if utils.ignore_branch_name(ref_name): | if utils.ignore_branch_name(ref_name): | ||||
continue | continue | ||||
remote_heads.add(ref_target) | remote_heads.add(ref_target) | ||||
logger.debug("local_heads_count=%s", len(local_heads)) | logger.debug("local_heads_count=%s", len(self.local_heads)) | ||||
logger.debug("remote_heads_count=%s", len(remote_heads)) | logger.debug("remote_heads_count=%s", len(remote_heads)) | ||||
wanted_refs = list(remote_heads - local_heads) | wanted_refs = list(remote_heads - self.local_heads) | ||||
logger.debug("wanted_refs_count=%s", len(wanted_refs)) | logger.debug("wanted_refs_count=%s", len(wanted_refs)) | ||||
if self.statsd is not None: | if self.statsd is not None: | ||||
self.statsd.histogram( | self.statsd.histogram( | ||||
"git_ignored_refs_percent", | "git_ignored_refs_percent", | ||||
len(remote_heads - set(refs.values())) / len(refs), | len(remote_heads - set(refs.values())) / len(refs), | ||||
tags={}, | tags={}, | ||||
) | ) | ||||
self.statsd.histogram( | self.statsd.histogram( | ||||
"git_known_refs_percent", | "git_known_refs_percent", | ||||
len(local_heads & remote_heads) / len(remote_heads), | len(self.local_heads & remote_heads) / len(remote_heads), | ||||
tags={}, | tags={}, | ||||
) | ) | ||||
return wanted_refs | return wanted_refs | ||||
@dataclass | @dataclass | ||||
class FetchPackReturn: | class FetchPackReturn: | ||||
remote_refs: Dict[bytes, HexBytes] | remote_refs: Dict[bytes, HexBytes] | ||||
▲ Show 20 Lines • Show All 444 Lines • Show Last 20 Lines |