diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -347,9 +347,9 @@ # with half the entries. self._cql_runner.directory_add_one(directory.id) - return {"directory:add": len(missing)} + return {"directory:add": len(directories)} - def directory_missing(self, directories): + def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]: return self._cql_runner.directory_missing(directories) def _join_dentry_to_content(self, dentry: DirectoryEntry) -> Dict[str, Any]: @@ -370,7 +370,9 @@ ret[key] = getattr(content, key) return ret - def _directory_ls(self, directory_id, recursive, prefix=b""): + def _directory_ls( + self, directory_id: Sha1Git, recursive: bool, prefix: bytes = b"" + ) -> Iterable[Dict[str, Any]]: if self.directory_missing([directory_id]): return rows = list(self._cql_runner.directory_entry_get([directory_id])) @@ -390,17 +392,21 @@ ret["target"], True, prefix + ret["name"] + b"/" ) - def directory_entry_get_by_path(self, directory, paths): + def directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes] + ) -> Optional[Dict[str, Any]]: return self._directory_entry_get_by_path(directory, paths, b"") - def _directory_entry_get_by_path(self, directory, paths, prefix): + def _directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes], prefix: bytes + ) -> Optional[Dict[str, Any]]: if not paths: - return + return None contents = list(self.directory_ls(directory)) if not contents: - return + return None def _get_entry(entries, name): """Finds the entry with the requested name, prepends the @@ -419,13 +425,15 @@ return first_item if not first_item or first_item["type"] != "dir": - return + return None return self._directory_entry_get_by_path( first_item["target"], paths[1:], prefix + paths[0] + b"/" ) - def directory_ls(self, directory, recursive=False): + def directory_ls( + self, directory: Sha1Git, recursive: bool = False + ) -> Iterable[Dict[str, Any]]: yield from self._directory_ls(directory, recursive) def directory_get_random(self) -> Sha1Git: diff --git a/swh/storage/in_memory.py b/swh/storage/in_memory.py --- a/swh/storage/in_memory.py +++ b/swh/storage/in_memory.py @@ -426,7 +426,7 @@ return {"directory:add": count} - def directory_missing(self, directories): + def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]: for id in directories: if id not in self._directories: yield id @@ -462,23 +462,29 @@ ret["target"], True, prefix + ret["name"] + b"/" ) - def directory_ls(self, directory, recursive=False): + def directory_ls( + self, directory: Sha1Git, recursive: bool = False + ) -> Iterable[Dict[str, Any]]: yield from self._directory_ls(directory, recursive) - def directory_entry_get_by_path(self, directory, paths): + def directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes] + ) -> Optional[Dict[str, Any]]: return self._directory_entry_get_by_path(directory, paths, b"") def directory_get_random(self) -> Sha1Git: return random.choice(list(self._directories)) - def _directory_entry_get_by_path(self, directory, paths, prefix): + def _directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes], prefix: bytes + ) -> Optional[Dict[str, Any]]: if not paths: - return + return None contents = list(self.directory_ls(directory)) if not contents: - return + return None def _get_entry(entries, name): for entry in entries: @@ -493,7 +499,7 @@ return first_item if not first_item or first_item["type"] != "dir": - return + return None return self._directory_entry_get_by_path( first_item["target"], paths[1:], prefix + paths[0] + b"/" diff --git a/swh/storage/interface.py b/swh/storage/interface.py --- a/swh/storage/interface.py +++ b/swh/storage/interface.py @@ -410,11 +410,11 @@ ... @remote_api_endpoint("directory/missing") - def directory_missing(self, directories): - """List directories missing from storage + def directory_missing(self, directories: List[Sha1Git]) -> Iterable[Sha1Git]: + """List directories missing from storage. Args: - directories (iterable): an iterable of directory ids + directories: list of directory ids Yields: missing directory ids @@ -423,33 +423,37 @@ ... @remote_api_endpoint("directory/ls") - def directory_ls(self, directory, recursive=False): - """Get entries for one directory. - - Args: - - directory: the directory to list entries from. - - recursive: if flag on, this list recursively from this directory. - - Returns: - List of entries for such directory. + def directory_ls( + self, directory: Sha1Git, recursive: bool = False + ) -> Iterable[Dict[str, Any]]: + """List entries for one directory. If `recursive=True`, names in the path of a dir/file not at the root are concatenated with a slash (`/`). + Args: + directory: the directory to list entries from. + recursive: if flag on, this list recursively from this directory. + + Yields: + directory entries for such directory. + """ ... @remote_api_endpoint("directory/path") - def directory_entry_get_by_path(self, directory, paths): + def directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes] + ) -> Optional[Dict[str, Any]]: """Get the directory entry (either file or dir) from directory with path. Args: - - directory: sha1 of the top level directory - - paths: path to lookup from the top level directory. From left + directory: directory id + paths: path to lookup from the top level directory. From left (top) to right (bottom). Returns: - The corresponding directory entry if found, None otherwise. + The corresponding directory entry as dict if found, None otherwise. """ ... diff --git a/swh/storage/storage.py b/swh/storage/storage.py --- a/swh/storage/storage.py +++ b/swh/storage/storage.py @@ -523,13 +523,17 @@ @timed @db_transaction_generator() - def directory_missing(self, directories, db=None, cur=None): + def directory_missing( + self, directories: List[Sha1Git], db=None, cur=None + ) -> Iterable[Sha1Git]: for obj in db.directory_missing_from_list(directories, cur): yield obj[0] @timed @db_transaction_generator(statement_timeout=20000) - def directory_ls(self, directory, recursive=False, db=None, cur=None): + def directory_ls( + self, directory: Sha1Git, recursive: bool = False, db=None, cur=None + ) -> Iterable[Dict[str, Any]]: if recursive: res_gen = db.directory_walk(directory, cur=cur) else: @@ -540,10 +544,11 @@ @timed @db_transaction(statement_timeout=2000) - def directory_entry_get_by_path(self, directory, paths, db=None, cur=None): + def directory_entry_get_by_path( + self, directory: Sha1Git, paths: List[bytes], db=None, cur=None + ) -> Optional[Dict[str, Any]]: res = db.directory_entry_get_by_path(directory, paths, cur) - if res: - return dict(zip(db.directory_ls_cols, res)) + return dict(zip(db.directory_ls_cols, res)) if res else None @timed @db_transaction()