Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/loader.py
| Show First 20 Lines • Show All 54 Lines • ▼ Show 20 Lines | ): | ||||
| self.incremental = incremental | self.incremental = incremental | ||||
| self.statsd = statsd | self.statsd = statsd | ||||
| if base_snapshots and incremental: | if base_snapshots and incremental: | ||||
| self.base_snapshots: List[Snapshot] = base_snapshots | self.base_snapshots: List[Snapshot] = base_snapshots | ||||
| else: | else: | ||||
| self.base_snapshots = [] | self.base_snapshots = [] | ||||
| self.heads: Set[HexBytes] = set() | # Cache existing heads | ||||
| self.local_heads: Set[HexBytes] = set() | |||||
| def get_parents(self, commit: bytes) -> List[bytes]: | heads_logger.debug("Heads known in the archive:") | ||||
| """This method should return the list of known parents""" | for base_snapshot in self.base_snapshots: | ||||
| return [] | for branch_name, branch in base_snapshot.branches.items(): | ||||
| if not branch or branch.target_type == TargetType.ALIAS: | |||||
| continue | |||||
| heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | |||||
| self.local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | |||||
| def graph_walker(self) -> ObjectStoreGraphWalker: | def graph_walker(self) -> ObjectStoreGraphWalker: | ||||
| return ObjectStoreGraphWalker(self.heads, self.get_parents) | return ObjectStoreGraphWalker(self.local_heads, get_parents=lambda commit: []) | ||||
| def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: | def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: | ||||
| """Get the list of bytehex sha1s that the git loader should fetch. | """Get the list of bytehex sha1s that the git loader should fetch. | ||||
| This compares the remote refs sent by the server with the base snapshot | This compares the remote refs sent by the server with the base snapshot | ||||
| provided by the loader. | provided by the loader. | ||||
| """ | """ | ||||
| if not refs: | if not refs: | ||||
| return [] | return [] | ||||
| if heads_logger.isEnabledFor(logging.DEBUG): | if heads_logger.isEnabledFor(logging.DEBUG): | ||||
| heads_logger.debug("Heads returned by the git remote:") | heads_logger.debug("Heads returned by the git remote:") | ||||
| for name, value in refs.items(): | for name, value in refs.items(): | ||||
| heads_logger.debug(" %r: %s", name, value.decode()) | heads_logger.debug(" %r: %s", name, value.decode()) | ||||
| heads_logger.debug("Heads known in the archive:") | |||||
| # Cache existing heads | |||||
| local_heads: Set[HexBytes] = set() | |||||
| for base_snapshot in self.base_snapshots: | |||||
| for branch_name, branch in base_snapshot.branches.items(): | |||||
| if not branch or branch.target_type == TargetType.ALIAS: | |||||
| continue | |||||
| heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | |||||
| local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | |||||
| self.heads = local_heads | |||||
| # Get the remote heads that we want to fetch | # Get the remote heads that we want to fetch | ||||
| remote_heads: Set[HexBytes] = set() | remote_heads: Set[HexBytes] = set() | ||||
| for ref_name, ref_target in refs.items(): | for ref_name, ref_target in refs.items(): | ||||
| if utils.ignore_branch_name(ref_name): | if utils.ignore_branch_name(ref_name): | ||||
| continue | continue | ||||
| remote_heads.add(ref_target) | remote_heads.add(ref_target) | ||||
| logger.debug("local_heads_count=%s", len(local_heads)) | logger.debug("local_heads_count=%s", len(self.local_heads)) | ||||
| logger.debug("remote_heads_count=%s", len(remote_heads)) | logger.debug("remote_heads_count=%s", len(remote_heads)) | ||||
| wanted_refs = list(remote_heads - local_heads) | wanted_refs = list(remote_heads - self.local_heads) | ||||
| logger.debug("wanted_refs_count=%s", len(wanted_refs)) | logger.debug("wanted_refs_count=%s", len(wanted_refs)) | ||||
| if self.statsd is not None: | if self.statsd is not None: | ||||
| self.statsd.histogram( | self.statsd.histogram( | ||||
| "git_ignored_refs_percent", | "git_ignored_refs_percent", | ||||
| len(remote_heads - set(refs.values())) / len(refs), | len(remote_heads - set(refs.values())) / len(refs), | ||||
| tags={}, | tags={}, | ||||
| ) | ) | ||||
| self.statsd.histogram( | self.statsd.histogram( | ||||
| "git_known_refs_percent", | "git_known_refs_percent", | ||||
| len(local_heads & remote_heads) / len(remote_heads), | len(self.local_heads & remote_heads) / len(remote_heads), | ||||
| tags={}, | tags={}, | ||||
| ) | ) | ||||
| return wanted_refs | return wanted_refs | ||||
| @dataclass | @dataclass | ||||
| class FetchPackReturn: | class FetchPackReturn: | ||||
| remote_refs: Dict[bytes, HexBytes] | remote_refs: Dict[bytes, HexBytes] | ||||
| ▲ Show 20 Lines • Show All 444 Lines • Show Last 20 Lines | |||||