Changeset View
Changeset View
Standalone View
Standalone View
swh/loader/git/loader.py
| Show First 20 Lines • Show All 288 Lines • ▼ Show 20 Lines | def fetch_data(self) -> bool: | ||||
| else: | else: | ||||
| self.pack_buffer = fetch_info.pack_buffer | self.pack_buffer = fetch_info.pack_buffer | ||||
| self.pack_size = fetch_info.pack_size | self.pack_size = fetch_info.pack_size | ||||
| self.remote_refs = fetch_info.remote_refs | self.remote_refs = fetch_info.remote_refs | ||||
| self.symbolic_refs = fetch_info.symbolic_refs | self.symbolic_refs = fetch_info.symbolic_refs | ||||
| self.ref_object_types = {sha1: None for sha1 in self.remote_refs.values()} | self.ref_object_types = {sha1: None for sha1 in self.remote_refs.values()} | ||||
| self.log.info( | logger.info( | ||||
| "Listed %d refs for repo %s", | "Listed %d refs for repo %s", | ||||
| len(self.remote_refs), | len(self.remote_refs), | ||||
| self.origin.url, | self.origin.url, | ||||
| extra={ | extra={ | ||||
| "swh_type": "git_repo_list_refs", | "swh_type": "git_repo_list_refs", | ||||
| "swh_repo": self.origin.url, | "swh_repo": self.origin.url, | ||||
| "swh_num_refs": len(self.remote_refs), | "swh_num_refs": len(self.remote_refs), | ||||
| }, | }, | ||||
| ▲ Show 20 Lines • Show All 49 Lines • ▼ Show 20 Lines | def get_contents(self) -> Iterable[BaseContent]: | ||||
| ) | ) | ||||
| def get_directories(self) -> Iterable[Directory]: | def get_directories(self) -> Iterable[Directory]: | ||||
| """Format the trees as swh directories""" | """Format the trees as swh directories""" | ||||
| for raw_obj in self.iter_objects(b"tree"): | for raw_obj in self.iter_objects(b"tree"): | ||||
| if raw_obj.id in self.ref_object_types: | if raw_obj.id in self.ref_object_types: | ||||
| self.ref_object_types[raw_obj.id] = TargetType.DIRECTORY | self.ref_object_types[raw_obj.id] = TargetType.DIRECTORY | ||||
| yield converters.dulwich_tree_to_directory(raw_obj, log=self.log) | yield converters.dulwich_tree_to_directory(raw_obj, log=logger) | ||||
| def get_revisions(self) -> Iterable[Revision]: | def get_revisions(self) -> Iterable[Revision]: | ||||
| """Format commits as swh revisions""" | """Format commits as swh revisions""" | ||||
| for raw_obj in self.iter_objects(b"commit"): | for raw_obj in self.iter_objects(b"commit"): | ||||
| if raw_obj.id in self.ref_object_types: | if raw_obj.id in self.ref_object_types: | ||||
| self.ref_object_types[raw_obj.id] = TargetType.REVISION | self.ref_object_types[raw_obj.id] = TargetType.REVISION | ||||
| yield converters.dulwich_commit_to_revision(raw_obj, log=self.log) | yield converters.dulwich_commit_to_revision(raw_obj, log=logger) | ||||
| def get_releases(self) -> Iterable[Release]: | def get_releases(self) -> Iterable[Release]: | ||||
| """Retrieve all the release objects from the git repository""" | """Retrieve all the release objects from the git repository""" | ||||
| for raw_obj in self.iter_objects(b"tag"): | for raw_obj in self.iter_objects(b"tag"): | ||||
| if raw_obj.id in self.ref_object_types: | if raw_obj.id in self.ref_object_types: | ||||
| self.ref_object_types[raw_obj.id] = TargetType.RELEASE | self.ref_object_types[raw_obj.id] = TargetType.RELEASE | ||||
| yield converters.dulwich_tag_to_release(raw_obj, log=self.log) | yield converters.dulwich_tag_to_release(raw_obj, log=logger) | ||||
| def get_snapshot(self) -> Snapshot: | def get_snapshot(self) -> Snapshot: | ||||
| """Get the snapshot for the current visit. | """Get the snapshot for the current visit. | ||||
| The main complexity of this function is mapping target objects to their | The main complexity of this function is mapping target objects to their | ||||
| types, as the `refs` dictionaries returned by the git server only give | types, as the `refs` dictionaries returned by the git server only give | ||||
| us the identifiers for the target objects, and not their types. | us the identifiers for the target objects, and not their types. | ||||
| ▲ Show 20 Lines • Show All 65 Lines • ▼ Show 20 Lines | def get_snapshot(self) -> Snapshot: | ||||
| ", ".join( | ", ".join( | ||||
| f"{name.decode()}: {hashutil.hash_to_hex(obj)}" | f"{name.decode()}: {hashutil.hash_to_hex(obj)}" | ||||
| for name, obj in unknown_objects.items() | for name, obj in unknown_objects.items() | ||||
| ) | ) | ||||
| ) | ) | ||||
| ) | ) | ||||
| utils.warn_dangling_branches( | utils.warn_dangling_branches( | ||||
| branches, dangling_branches, self.log, self.origin_url | branches, dangling_branches, logger, self.origin_url | ||||
| ) | ) | ||||
| self.snapshot = Snapshot(branches=branches) | self.snapshot = Snapshot(branches=branches) | ||||
| return self.snapshot | return self.snapshot | ||||
| def load_status(self) -> Dict[str, Any]: | def load_status(self) -> Dict[str, Any]: | ||||
| """The load was eventful if the current snapshot is different to | """The load was eventful if the current snapshot is different to | ||||
| the one we retrieved at the beginning of the run""" | the one we retrieved at the beginning of the run""" | ||||
| Show All 35 Lines | |||||