Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/loader.py
| Show All 36 Lines | |||||
| from . import converters, dumb, utils | from . import converters, dumb, utils | ||||
| from .base import BaseGitLoader | from .base import BaseGitLoader | ||||
| from .utils import HexBytes | from .utils import HexBytes | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| heads_logger = logger.getChild("refs") | heads_logger = logger.getChild("refs") | ||||
| DEFAULT_NUMBER_IDS_TO_FETCH = 200 | |||||
| def do_print_progress(msg: bytes) -> None: | |||||
| sys.stderr.buffer.write(msg) | |||||
ardumont: Do we need this?
I'd very much see this as silently ignored... | |||||
Done Inline Actions
nvm, let's keep it. ardumont: > Do we need this?
> I'd very much see this as silently ignored...
nvm, let's keep it. | |||||
| sys.stderr.flush() | |||||
| class RepoRepresentation: | class RepoRepresentation: | ||||
| """Repository representation for a Software Heritage origin.""" | """Repository representation for a Software Heritage origin.""" | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| storage, | storage, | ||||
| base_snapshots: List[Snapshot] = None, | base_snapshots: Optional[List[Snapshot]] = None, | ||||
| incremental: bool = True, | incremental: bool = True, | ||||
| statsd: Statsd = None, | statsd: Optional[Statsd] = None, | ||||
| limit: int = DEFAULT_NUMBER_IDS_TO_FETCH, | |||||
| ): | ): | ||||
| self.storage = storage | self.storage = storage | ||||
| self.incremental = incremental | self.incremental = incremental | ||||
| self.statsd = statsd | self.statsd = statsd | ||||
| if base_snapshots and incremental: | if base_snapshots and incremental: | ||||
| self.base_snapshots: List[Snapshot] = base_snapshots | self.base_snapshots: List[Snapshot] = base_snapshots | ||||
| else: | else: | ||||
| self.base_snapshots = [] | self.base_snapshots = [] | ||||
| # Cache existing heads | # Cache existing heads | ||||
| self.local_heads: Set[HexBytes] = set() | self.local_heads: Set[HexBytes] = set() | ||||
| heads_logger.debug("Heads known in the archive:") | heads_logger.debug("Heads known in the archive:") | ||||
| for base_snapshot in self.base_snapshots: | for base_snapshot in self.base_snapshots: | ||||
| for branch_name, branch in base_snapshot.branches.items(): | for branch_name, branch in base_snapshot.branches.items(): | ||||
| if not branch or branch.target_type == TargetType.ALIAS: | if not branch or branch.target_type == TargetType.ALIAS: | ||||
| continue | continue | ||||
| heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | heads_logger.debug(" %r: %s", branch_name, branch.target.hex()) | ||||
| self.local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | self.local_heads.add(HexBytes(hashutil.hash_to_bytehex(branch.target))) | ||||
| def graph_walker(self) -> ObjectStoreGraphWalker: | self.heads: Set[HexBytes] = set() | ||||
| return ObjectStoreGraphWalker(self.local_heads, get_parents=lambda commit: []) | self.wanted_refs: Optional[List[HexBytes]] = None | ||||
| self.walker = ObjectStoreGraphWalker(self.heads, lambda commit: []) | |||||
| def determine_wants(self, refs: Dict[bytes, HexBytes]) -> List[HexBytes]: | # Pagination index | ||||
| self.index: int = 0 | |||||
| self.limit = limit | |||||
| self.previous_refs: List[HexBytes] = [] | |||||
| def more_refs_to_fetch(self) -> bool: | |||||
| """Did we fetch all wanted refs?""" | |||||
| return self.wanted_refs is not None and self.index > len(self.wanted_refs) | |||||
| def graph_walker(self): | |||||
| return self.walker | |||||
| def determine_wants( | |||||
| self, refs: Dict[bytes, HexBytes], depth=None | |||||
| ) -> List[HexBytes]: | |||||
| """Get the list of bytehex sha1s that the git loader should fetch. | """Get the list of bytehex sha1s that the git loader should fetch. | ||||
| This compares the remote refs sent by the server with the base snapshot | This compares the remote refs sent by the server with the base snapshot | ||||
| provided by the loader. | provided by the loader. | ||||
| """ | """ | ||||
| if not refs: | if not refs: | ||||
| return [] | return [] | ||||
| if not self.wanted_refs: | |||||
| # We'll compute all wanted_refs to ingest but we'll return it by batch of | |||||
| # limit | |||||
| if heads_logger.isEnabledFor(logging.DEBUG): | if heads_logger.isEnabledFor(logging.DEBUG): | ||||
| heads_logger.debug("Heads returned by the git remote:") | heads_logger.debug("Heads returned by the git remote:") | ||||
| for name, value in refs.items(): | for name, value in refs.items(): | ||||
| heads_logger.debug(" %r: %s", name, value.decode()) | heads_logger.debug(" %r: %s", name, value.decode()) | ||||
| # Get the remote heads that we want to fetch | # Get the remote heads that we want to fetch | ||||
| remote_heads: Set[HexBytes] = set() | remote_heads: Set[HexBytes] = set() | ||||
| for ref_name, ref_target in refs.items(): | for ref_name, ref_target in refs.items(): | ||||
| if utils.ignore_branch_name(ref_name): | if utils.ignore_branch_name(ref_name): | ||||
| continue | continue | ||||
| remote_heads.add(ref_target) | remote_heads.add(ref_target) | ||||
| logger.debug("local_heads_count=%s", len(self.local_heads)) | logger.debug("local_heads_count=%s", len(self.local_heads)) | ||||
| logger.debug("remote_heads_count=%s", len(remote_heads)) | logger.debug("remote_heads_count=%s", len(remote_heads)) | ||||
| wanted_refs = list(remote_heads - self.local_heads) | wanted_refs = list(remote_heads - self.local_heads) | ||||
| logger.debug("wanted_refs_count=%s", len(wanted_refs)) | logger.debug("wanted_refs_count=%s", len(wanted_refs)) | ||||
| if self.statsd is not None: | if self.statsd is not None: | ||||
| self.statsd.histogram( | self.statsd.histogram( | ||||
| "git_ignored_refs_percent", | "git_ignored_refs_percent", | ||||
| len(remote_heads - set(refs.values())) / len(refs), | len(remote_heads - set(refs.values())) / len(refs), | ||||
| tags={}, | tags={}, | ||||
| ) | ) | ||||
| self.statsd.histogram( | self.statsd.histogram( | ||||
| "git_known_refs_percent", | "git_known_refs_percent", | ||||
| len(self.local_heads & remote_heads) / len(remote_heads), | len(self.local_heads & remote_heads) / len(remote_heads), | ||||
| tags={}, | tags={}, | ||||
| ) | ) | ||||
| return wanted_refs | # Determine all refs we want to ingest | ||||
| self.wanted_refs = wanted_refs | |||||
| # We're gonna ingest them in smaller interval of refs | |||||
| start = self.index | |||||
| self.index += self.limit | |||||
| assert self.wanted_refs is not None | |||||
| asked_refs = self.wanted_refs[start : self.index] | |||||
| if start > 0: | |||||
| # Previous refs was already walked so we can remove them from the next walk | |||||
| # iteration to avoid processing them again | |||||
| self.walker.heads.update(self.previous_refs) | |||||
| self.previous_refs = asked_refs | |||||
| logger.debug("asked_refs_count=%s", len(asked_refs)) | |||||
| return asked_refs | |||||
| @dataclass | @dataclass | ||||
| class FetchPackReturn: | class FetchPackReturn: | ||||
| remote_refs: Dict[bytes, HexBytes] | remote_refs: Dict[bytes, HexBytes] | ||||
| symbolic_refs: Dict[bytes, HexBytes] | symbolic_refs: Dict[bytes, HexBytes] | ||||
| pack_buffer: SpooledTemporaryFile | pack_buffer: SpooledTemporaryFile | ||||
| pack_size: int | pack_size: int | ||||
| continue_loading: bool | |||||
| class GitLoader(BaseGitLoader): | class GitLoader(BaseGitLoader): | ||||
| """A bulk loader for a git repository | """A bulk loader for a git repository | ||||
| Emits the following statsd stats: | Emits the following statsd stats: | ||||
| * increments ``swh_loader_git`` | * increments ``swh_loader_git`` | ||||
| Show All 20 Lines | class GitLoader(BaseGitLoader): | ||||
| def __init__( | def __init__( | ||||
| self, | self, | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| url: str, | url: str, | ||||
| incremental: bool = True, | incremental: bool = True, | ||||
| repo_representation: Type[RepoRepresentation] = RepoRepresentation, | repo_representation: Type[RepoRepresentation] = RepoRepresentation, | ||||
| pack_size_bytes: int = 4 * 1024 * 1024 * 1024, | pack_size_bytes: int = 4 * 1024 * 1024 * 1024, | ||||
| temp_file_cutoff: int = 100 * 1024 * 1024, | temp_file_cutoff: int = 100 * 1024 * 1024, | ||||
| packfile_chunk_size: int = DEFAULT_NUMBER_IDS_TO_FETCH, | |||||
| **kwargs: Any, | **kwargs: Any, | ||||
| ): | ): | ||||
| """Initialize the bulk updater. | """Initialize the bulk updater. | ||||
| Args: | Args: | ||||
| repo_representation: swh's repository representation | repo_representation: swh's repository representation | ||||
| which is in charge of filtering between known and remote | which is in charge of filtering between known and remote | ||||
| data. | data. | ||||
| ... | ... | ||||
| incremental: If True, the default, this starts from the last known snapshot | incremental: If True, the default, this starts from the last known snapshot | ||||
| (if any) references. Otherwise, this loads the full repository. | (if any) references. Otherwise, this loads the full repository. | ||||
| """ | """ | ||||
| super().__init__(storage=storage, origin_url=url, **kwargs) | super().__init__(storage=storage, origin_url=url, **kwargs) | ||||
| # check if repository only supports git dumb transfer protocol, | |||||
| # fetched pack file will be empty in that case as dulwich do | |||||
| # not support it and do not fetch any refs | |||||
| logger.debug("Transport url to communicate with server: %s", url) | |||||
| self.client, self.path = dulwich.client.get_transport_and_path( | |||||
| url, thin_packs=False | |||||
| ) | |||||
| logger.debug("Client %s to fetch pack at %s", self.client, self.path) | |||||
| self.dumb = url.startswith("http") and getattr(self.client, "dumb", False) | |||||
| # will create partial snapshot alongside fetching mutliple packfiles | |||||
| self.create_partial_snapshot = not self.dumb and incremental | |||||
| self.incremental = incremental | self.incremental = incremental | ||||
| self.repo_representation = repo_representation | self.repo_representation = repo_representation | ||||
| self.pack_size_bytes = pack_size_bytes | self.pack_size_bytes = pack_size_bytes | ||||
| self.temp_file_cutoff = temp_file_cutoff | self.temp_file_cutoff = temp_file_cutoff | ||||
| # state initialized in fetch_data | # state initialized in fetch_data | ||||
| self.remote_refs: Dict[bytes, HexBytes] = {} | self.remote_refs: Dict[bytes, HexBytes] = {} | ||||
| self.symbolic_refs: Dict[bytes, HexBytes] = {} | self.symbolic_refs: Dict[bytes, HexBytes] = {} | ||||
| self.ref_object_types: Dict[bytes, Optional[TargetType]] = {} | self.ref_object_types: Dict[bytes, Optional[TargetType]] = {} | ||||
| self.packfile_chunk_size = packfile_chunk_size | |||||
| def fetch_pack_from_origin( | def fetch_pack_from_origin( | ||||
| self, | self, | ||||
| origin_url: str, | origin_url: str, | ||||
| base_repo: RepoRepresentation, | base_repo: RepoRepresentation, | ||||
| do_activity: Callable[[bytes], None], | do_activity: Callable[[bytes], None], | ||||
| ) -> FetchPackReturn: | ) -> FetchPackReturn: | ||||
| """Fetch a pack from the origin""" | """Fetch a pack from the origin""" | ||||
| pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff) | pack_buffer = SpooledTemporaryFile(max_size=self.temp_file_cutoff) | ||||
| transport_url = origin_url | |||||
| logger.debug("Transport url to communicate with server: %s", transport_url) | |||||
| client, path = dulwich.client.get_transport_and_path( | |||||
| transport_url, thin_packs=False | |||||
| ) | |||||
| logger.debug("Client %s to fetch pack at %s", client, path) | |||||
| size_limit = self.pack_size_bytes | size_limit = self.pack_size_bytes | ||||
| def do_pack(data: bytes) -> None: | def do_pack(data: bytes) -> None: | ||||
| cur_size = pack_buffer.tell() | cur_size = pack_buffer.tell() | ||||
| would_write = len(data) | would_write = len(data) | ||||
| if cur_size + would_write > size_limit: | if cur_size + would_write > size_limit: | ||||
| raise IOError( | raise IOError( | ||||
| f"Pack file too big for repository {origin_url}, " | f"Pack file too big for repository {origin_url}, " | ||||
| f"limit is {size_limit} bytes, current size is {cur_size}, " | f"limit is {size_limit} bytes, current size is {cur_size}, " | ||||
| f"would write {would_write}" | f"would write {would_write}" | ||||
| ) | ) | ||||
| pack_buffer.write(data) | pack_buffer.write(data) | ||||
| pack_result = client.fetch_pack( | pack_result = self.client.fetch_pack( | ||||
| path, | self.path, | ||||
| base_repo.determine_wants, | base_repo.determine_wants, | ||||
| base_repo.graph_walker(), | base_repo.graph_walker(), | ||||
| do_pack, | do_pack, | ||||
| progress=do_activity, | progress=do_activity, | ||||
| ) | ) | ||||
| remote_refs = pack_result.refs or {} | remote_refs = pack_result.refs or {} | ||||
| symbolic_refs = pack_result.symrefs or {} | symbolic_refs = pack_result.symrefs or {} | ||||
| pack_buffer.flush() | pack_buffer.flush() | ||||
| pack_size = pack_buffer.tell() | pack_size = pack_buffer.tell() | ||||
| pack_buffer.seek(0) | pack_buffer.seek(0) | ||||
| logger.debug("fetched_pack_size=%s", pack_size) | logger.debug("fetched_pack_size=%s", pack_size) | ||||
| # check if repository only supports git dumb transfer protocol, | |||||
| # fetched pack file will be empty in that case as dulwich do | |||||
| # not support it and do not fetch any refs | |||||
| self.dumb = transport_url.startswith("http") and getattr(client, "dumb", False) | |||||
| return FetchPackReturn( | return FetchPackReturn( | ||||
| remote_refs=utils.filter_refs(remote_refs), | remote_refs=remote_refs, | ||||
| symbolic_refs=utils.filter_refs(symbolic_refs), | symbolic_refs=symbolic_refs, | ||||
| pack_buffer=pack_buffer, | pack_buffer=pack_buffer, | ||||
| pack_size=pack_size, | pack_size=pack_size, | ||||
| continue_loading=not self.base_repo.more_refs_to_fetch(), | |||||
| ) | ) | ||||
| def get_full_snapshot(self, origin_url) -> Optional[Snapshot]: | def get_full_snapshot(self, origin_url) -> Optional[Snapshot]: | ||||
| return snapshot_get_latest(self.storage, origin_url) | return snapshot_get_latest(self.storage, origin_url) | ||||
| def prepare(self) -> None: | def prepare(self) -> None: | ||||
| assert self.origin is not None | assert self.origin is not None | ||||
| Show All 24 Lines | def prepare(self) -> None: | ||||
| if parent_snapshot is not None: | if parent_snapshot is not None: | ||||
| self.statsd.constant_tags["has_parent_snapshot"] = True | self.statsd.constant_tags["has_parent_snapshot"] = True | ||||
| self.base_snapshots.append(parent_snapshot) | self.base_snapshots.append(parent_snapshot) | ||||
| # Increments a metric with full name 'swh_loader_git'; which is useful to | # Increments a metric with full name 'swh_loader_git'; which is useful to | ||||
| # count how many runs of the loader are with each incremental mode | # count how many runs of the loader are with each incremental mode | ||||
| self.statsd.increment("git_total", tags={}) | self.statsd.increment("git_total", tags={}) | ||||
| def fetch_data(self) -> bool: | self.base_repo = self.repo_representation( | ||||
| assert self.origin is not None | |||||
| base_repo = self.repo_representation( | |||||
| storage=self.storage, | storage=self.storage, | ||||
| base_snapshots=self.base_snapshots, | base_snapshots=self.base_snapshots, | ||||
| incremental=self.incremental, | incremental=self.incremental, | ||||
| statsd=self.statsd, | statsd=self.statsd, | ||||
| limit=self.packfile_chunk_size, | |||||
| ) | ) | ||||
| def do_progress(msg: bytes) -> None: | def fetch_data(self) -> bool: | ||||
| sys.stderr.buffer.write(msg) | continue_loading = False | ||||
| sys.stderr.flush() | assert self.origin is not None | ||||
| try: | try: | ||||
| fetch_info = self.fetch_pack_from_origin( | fetch_info = self.fetch_pack_from_origin( | ||||
| self.origin.url, base_repo, do_progress | self.origin.url, self.base_repo, do_print_progress | ||||
| ) | ) | ||||
| continue_loading = fetch_info.continue_loading | |||||
| except (dulwich.client.HTTPUnauthorized, NotGitRepository) as e: | except (dulwich.client.HTTPUnauthorized, NotGitRepository) as e: | ||||
| raise NotFound(e) | raise NotFound(e) | ||||
| except GitProtocolError as e: | except GitProtocolError as e: | ||||
| # unfortunately, that kind of error is not specific to a not found | # unfortunately, that kind of error is not specific to a not found | ||||
| # scenario... It depends on the value of message within the exception. | # scenario... It depends on the value of message within the exception. | ||||
| for msg in [ | for msg in [ | ||||
| "Repository unavailable", # e.g DMCA takedown | "Repository unavailable", # e.g DMCA takedown | ||||
| "Repository not found", | "Repository not found", | ||||
| "unexpected http resp 401", | "unexpected http resp 401", | ||||
| ]: | ]: | ||||
| if msg in e.args[0]: | if msg in e.args[0]: | ||||
| raise NotFound(e) | raise NotFound(e) | ||||
| # otherwise transmit the error | # otherwise transmit the error | ||||
| raise | raise | ||||
| except (AttributeError, NotImplementedError, ValueError): | except (AttributeError, NotImplementedError, ValueError): | ||||
| # with old dulwich versions, those exceptions types can be raised | # with old dulwich versions, those exceptions types can be raised | ||||
| # by the fetch_pack operation when encountering a repository with | # by the fetch_pack operation when encountering a repository with | ||||
| # dumb transfer protocol so we check if the repository supports it | # dumb transfer protocol so we check if the repository supports it | ||||
| # here to continue the loading if it is the case | # here to continue the loading if it is the case | ||||
| self.dumb = dumb.check_protocol(self.origin.url) | self.dumb = dumb.check_protocol(self.origin.url) | ||||
| if not self.dumb: | if not self.dumb: | ||||
| raise | raise | ||||
| logger.debug( | |||||
| "Protocol used for communication: %s", "dumb" if self.dumb else "smart" | |||||
| ) | |||||
| if self.dumb: | if self.dumb: | ||||
| self.dumb_fetcher = dumb.GitObjectsFetcher(self.origin.url, base_repo) | protocol = "dumb" | ||||
| self.dumb_fetcher = dumb.GitObjectsFetcher(self.origin.url, self.base_repo) | |||||
| self.dumb_fetcher.fetch_object_ids() | self.dumb_fetcher.fetch_object_ids() | ||||
| self.remote_refs = utils.filter_refs(self.dumb_fetcher.refs) | remote_refs = self.dumb_fetcher.refs | ||||
| self.symbolic_refs = utils.filter_refs(self.dumb_fetcher.head) | symbolic_refs = self.dumb_fetcher.head | ||||
| else: | else: | ||||
| protocol = "smart" | |||||
| self.pack_buffer = fetch_info.pack_buffer | self.pack_buffer = fetch_info.pack_buffer | ||||
| self.pack_size = fetch_info.pack_size | self.pack_size = fetch_info.pack_size | ||||
| self.remote_refs = fetch_info.remote_refs | remote_refs = fetch_info.remote_refs | ||||
| self.symbolic_refs = fetch_info.symbolic_refs | symbolic_refs = fetch_info.symbolic_refs | ||||
| self.ref_object_types = {sha1: None for sha1 in self.remote_refs.values()} | logger.debug("Protocol used for communication: %s", protocol) | ||||
| # So the partial snapshot and the final ones creates the full branches | |||||
| self.remote_refs.update(utils.filter_refs(remote_refs)) | |||||
| self.symbolic_refs.update(utils.filter_refs(symbolic_refs)) | |||||
| for sha1 in self.remote_refs.values(): | |||||
| if sha1 in self.ref_object_types: | |||||
| continue | |||||
| self.ref_object_types[sha1] = None | |||||
| logger.info( | logger.info( | ||||
| "Listed %d refs for repo %s", | "Listed %d refs for repo %s", | ||||
| len(self.remote_refs), | len(self.remote_refs), | ||||
| self.origin.url, | self.origin.url, | ||||
| extra={ | extra={ | ||||
| "swh_type": "git_repo_list_refs", | "swh_type": "git_repo_list_refs", | ||||
| "swh_repo": self.origin.url, | "swh_repo": self.origin.url, | ||||
| "swh_num_refs": len(self.remote_refs), | "swh_num_refs": len(self.remote_refs), | ||||
| }, | }, | ||||
| ) | ) | ||||
| # No more data to fetch | return continue_loading | ||||
| return False | |||||
| def save_data(self) -> None: | def save_data(self) -> None: | ||||
| """Store a pack for archival""" | """Store a pack for archival""" | ||||
| assert isinstance(self.visit_date, datetime.datetime) | assert isinstance(self.visit_date, datetime.datetime) | ||||
| write_size = 8192 | write_size = 8192 | ||||
| pack_dir = self.get_save_data_path() | pack_dir = self.get_save_data_path() | ||||
| pack_name = "%s.pack" % self.visit_date.isoformat() | pack_name = "%s.pack" % self.visit_date.isoformat() | ||||
| refs_name = "%s.refs" % self.visit_date.isoformat() | refs_name = "%s.refs" % self.visit_date.isoformat() | ||||
| with open(os.path.join(pack_dir, pack_name), "xb") as f: | with open(os.path.join(pack_dir, pack_name), "xb") as f: | ||||
| self.pack_buffer.seek(0) | self.pack_buffer.seek(0) | ||||
| while True: | while True: | ||||
| r = self.pack_buffer.read(write_size) | r = self.pack_buffer.read(write_size) | ||||
| if not r: | if not r: | ||||
| break | break | ||||
| f.write(r) | f.write(r) | ||||
| self.pack_buffer.seek(0) | self.pack_buffer.seek(0) | ||||
| with open(os.path.join(pack_dir, refs_name), "xb") as f: | with open(os.path.join(pack_dir, refs_name), "xb") as f: | ||||
| pickle.dump(self.remote_refs, f) | pickle.dump(self.remote_refs, f) | ||||
| def build_partial_snapshot(self) -> Optional[Snapshot]: | |||||
| # Current implementation makes it a simple call to existing :meth:`get_snapshot` | |||||
| return self.get_snapshot() | |||||
| def store_data(self): | |||||
| """Override the default implementation so we make sure to close the pack_buffer | |||||
| if we use one in between loop (dumb loader does not actually one for example). | |||||
| """ | |||||
| super().store_data() | |||||
| if not self.dumb: | |||||
| self.pack_buffer.close() | |||||
| def iter_objects(self, object_type: bytes) -> Iterator[ShaFile]: | def iter_objects(self, object_type: bytes) -> Iterator[ShaFile]: | ||||
| """Read all the objects of type `object_type` from the packfile""" | """Read all the objects of type `object_type` from the packfile""" | ||||
| if self.dumb: | if self.dumb: | ||||
| yield from self.dumb_fetcher.iter_objects(object_type) | yield from self.dumb_fetcher.iter_objects(object_type) | ||||
| else: | else: | ||||
| self.pack_buffer.seek(0) | self.pack_buffer.seek(0) | ||||
| count = 0 | count = 0 | ||||
| for obj in PackInflater.for_pack_data( | for obj in PackInflater.for_pack_data( | ||||
| ▲ Show 20 Lines • Show All 151 Lines • ▼ Show 20 Lines | def get_snapshot(self) -> Snapshot: | ||||
| targets_unknown = missing | targets_unknown = missing | ||||
| if not targets_unknown: | if not targets_unknown: | ||||
| break | break | ||||
| if unknown_objects: | if unknown_objects: | ||||
| # This object was referenced by the server; We did not fetch | # This object was referenced by the server; We did not fetch | ||||
| # it, and we do not know it from the previous snapshot. This is | # it, and we do not know it from the previous snapshot. This is | ||||
| # likely a bug in the loader. | # possible as we allow partial snapshot | ||||
| raise RuntimeError( | logger.warning( | ||||
| "Unknown objects referenced by remote refs: %s" | "Unknown objects referenced by remote refs: %s", | ||||
| % ( | ( | ||||
| ", ".join( | ", ".join( | ||||
| f"{name.decode()}: {hashutil.hash_to_hex(obj)}" | f"{name.decode()}: {hashutil.hash_to_hex(obj)}" | ||||
| for name, obj in unknown_objects.items() | for name, obj in unknown_objects.items() | ||||
| ) | ) | ||||
| ) | ), | ||||
Done Inline ActionsThat warning becomes actually tedious (and possibly strenuous for elasticsearch)... As it will eventually be resolved if we get to the bottom of the repository, do we keep that log instruction? [1] P1194 ardumont: That warning becomes actually tedious (and possibly strenuous for elasticsearch)...
On staging… | |||||
| ) | ) | ||||
| utils.warn_dangling_branches( | utils.warn_dangling_branches( | ||||
| branches, dangling_branches, logger, self.origin.url | branches, dangling_branches, logger, self.origin.url | ||||
| ) | ) | ||||
| self.snapshot = Snapshot(branches=branches) | self.snapshot = Snapshot(branches=branches) | ||||
| return self.snapshot | return self.snapshot | ||||
| ▲ Show 20 Lines • Show All 44 Lines • Show Last 20 Lines | |||||
Do we need this?
I'd very much see this as silently ignored...