Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show First 20 Lines • Show All 508 Lines • ▼ Show 20 Lines | def directory_add(self, directories: List[Directory]) -> Dict[str, int]: | ||||
| self._cql_runner.directory_add_one(DirectoryRow(id=directory.id)) | self._cql_runner.directory_add_one(DirectoryRow(id=directory.id)) | ||||
| return {"directory:add": len(directories)} | return {"directory:add": len(directories)} | ||||
| @timed | @timed | ||||
| def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]: | def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]: | ||||
| return self._cql_runner.directory_missing(directories) | return self._cql_runner.directory_missing(directories) | ||||
| def _join_dentry_to_content(self, dentry: DirectoryEntry) -> Dict[str, Any]: | def _join_dentry_to_content( | ||||
| contents: Union[List[Content], List[SkippedContentRow]] | self, dentry: DirectoryEntry, contents: List[Content] | ||||
| ) -> Dict[str, Any]: | |||||
| content: Union[None, Content, SkippedContentRow] | |||||
| keys = ( | keys = ( | ||||
| "status", | "status", | ||||
| "sha1", | "sha1", | ||||
| "sha1_git", | "sha1_git", | ||||
| "sha256", | "sha256", | ||||
| "length", | "length", | ||||
| ) | ) | ||||
| ret = dict.fromkeys(keys) | ret = dict.fromkeys(keys) | ||||
| ret.update(dentry.to_dict()) | ret.update(dentry.to_dict()) | ||||
| if ret["type"] == "file": | if ret["type"] == "file": | ||||
| contents = self.content_find({"sha1_git": ret["target"]}) | for content in contents: | ||||
| if not contents: | if dentry.target == content.sha1_git: | ||||
| break | |||||
| else: | |||||
| tokens = list( | tokens = list( | ||||
| self._cql_runner.skipped_content_get_tokens_from_single_hash( | self._cql_runner.skipped_content_get_tokens_from_single_hash( | ||||
| "sha1_git", ret["target"] | "sha1_git", ret["target"] | ||||
| ) | ) | ||||
| ) | ) | ||||
| if tokens: | if tokens: | ||||
| contents = list( | content = list( | ||||
| self._cql_runner.skipped_content_get_from_token(tokens[0]) | self._cql_runner.skipped_content_get_from_token(tokens[0]) | ||||
| ) | )[0] | ||||
| if contents: | else: | ||||
| content = contents[0] | content = None | ||||
| if content: | |||||
| for key in keys: | for key in keys: | ||||
| ret[key] = getattr(content, key) | ret[key] = getattr(content, key) | ||||
| return ret | return ret | ||||
| def _directory_ls( | def _directory_ls( | ||||
| self, directory_id: Sha1Git, recursive: bool, prefix: bytes = b"" | self, directory_id: Sha1Git, recursive: bool, prefix: bytes = b"" | ||||
| ) -> Iterable[Dict[str, Any]]: | ) -> Iterable[Dict[str, Any]]: | ||||
| if self.directory_missing([directory_id]): | if self.directory_missing([directory_id]): | ||||
| return | return | ||||
| rows = list(self._cql_runner.directory_entry_get([directory_id])) | rows = list(self._cql_runner.directory_entry_get([directory_id])) | ||||
| # TODO: dedup to be fast in case the directory contains the same subdir/file | |||||
| # multiple times | |||||
| contents = self._content_find_many([{"sha1_git": row.target} for row in rows]) | |||||
| for row in rows: | for row in rows: | ||||
| entry_d = row.to_dict() | entry_d = row.to_dict() | ||||
| # Build and yield the directory entry dict | # Build and yield the directory entry dict | ||||
| del entry_d["directory_id"] | del entry_d["directory_id"] | ||||
| entry = DirectoryEntry.from_dict(entry_d) | entry = DirectoryEntry.from_dict(entry_d) | ||||
| ret = self._join_dentry_to_content(entry) | ret = self._join_dentry_to_content(entry, contents) | ||||
| ret["name"] = prefix + ret["name"] | ret["name"] = prefix + ret["name"] | ||||
| ret["dir_id"] = directory_id | ret["dir_id"] = directory_id | ||||
| yield ret | yield ret | ||||
| if recursive and ret["type"] == "dir": | if recursive and ret["type"] == "dir": | ||||
| yield from self._directory_ls( | yield from self._directory_ls( | ||||
| ret["target"], True, prefix + ret["name"] + b"/" | ret["target"], True, prefix + ret["name"] + b"/" | ||||
| ) | ) | ||||
| ▲ Show 20 Lines • Show All 1,110 Lines • Show Last 20 Lines | |||||