diff --git a/swh/storage/buffer.py b/swh/storage/buffer.py --- a/swh/storage/buffer.py +++ b/swh/storage/buffer.py @@ -90,7 +90,7 @@ raise AttributeError(key) return getattr(self.storage, key) - def content_add(self, contents: Sequence[Content]) -> Dict: + def content_add(self, contents: Sequence[Content]) -> Dict[str, int]: """Push contents to write to the storage in the buffer. Following policies apply: @@ -112,7 +112,7 @@ return stats - def skipped_content_add(self, contents: Sequence[SkippedContent]) -> Dict: + def skipped_content_add(self, contents: Sequence[SkippedContent]) -> Dict[str, int]: return self.object_add( contents, object_type="skipped_content", diff --git a/swh/storage/cassandra/storage.py b/swh/storage/cassandra/storage.py --- a/swh/storage/cassandra/storage.py +++ b/swh/storage/cassandra/storage.py @@ -117,7 +117,7 @@ if getattr(row, algo) == hash_: yield row - def _content_add(self, contents: List[Content], with_data: bool) -> Dict: + def _content_add(self, contents: List[Content], with_data: bool) -> Dict[str, int]: # Filter-out content already in the database. contents = [ c for c in contents if not self._cql_runner.content_get_from_pk(c.to_dict()) @@ -192,7 +192,7 @@ return summary - def content_add(self, content: List[Content]) -> Dict: + def content_add(self, content: List[Content]) -> Dict[str, int]: to_add = { (c.sha1, c.sha1_git, c.sha256, c.blake2s256): c for c in content }.values() @@ -206,7 +206,7 @@ "content_update is not supported by the Cassandra backend" ) - def content_add_metadata(self, content: List[Content]) -> Dict: + def content_add_metadata(self, content: List[Content]) -> Dict[str, int]: return self._content_add(content, with_data=False) def content_get_data(self, content: Sha1) -> Optional[bytes]: @@ -321,7 +321,7 @@ assert content, "Could not find any content" return content.sha1_git - def _skipped_content_add(self, contents: List[SkippedContent]) -> Dict: + def _skipped_content_add(self, contents: List[SkippedContent]) -> Dict[str, int]: # Filter-out content already in the database. contents = [ c @@ -346,7 +346,7 @@ return {"skipped_content:add": len(contents)} - def skipped_content_add(self, content: List[SkippedContent]) -> Dict: + def skipped_content_add(self, content: List[SkippedContent]) -> Dict[str, int]: contents = [attr.evolve(c, ctime=now()) for c in content] return self._skipped_content_add(contents) @@ -357,7 +357,7 @@ if not self._cql_runner.skipped_content_get_from_pk(content): yield {algo: content[algo] for algo in DEFAULT_ALGORITHMS} - def directory_add(self, directories: List[Directory]) -> Dict: + def directory_add(self, directories: List[Directory]) -> Dict[str, int]: to_add = {d.id: d for d in directories}.values() # Filter out directories that are already inserted. missing = self.directory_missing([dir_.id for dir_ in to_add]) @@ -482,7 +482,7 @@ assert directory, "Could not find any directory" return directory.id - def revision_add(self, revisions: List[Revision]) -> Dict: + def revision_add(self, revisions: List[Revision]) -> Dict[str, int]: # Filter-out revisions already in the database to_add = {r.id: r for r in revisions}.values() missing = self.revision_missing([rev.id for rev in to_add]) @@ -594,7 +594,7 @@ assert revision, "Could not find any revision" return revision.id - def release_add(self, releases: List[Release]) -> Dict: + def release_add(self, releases: List[Release]) -> Dict[str, int]: to_add = {r.id: r for r in releases}.values() missing = set(self.release_missing([rel.id for rel in to_add])) releases = [rel for rel in to_add if rel.id in missing] @@ -623,7 +623,7 @@ assert release, "Could not find any release" return release.id - def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: + def snapshot_add(self, snapshots: List[Snapshot]) -> Dict[str, int]: to_add = {s.id: s for s in snapshots}.values() missing = self._cql_runner.snapshot_missing([snp.id for snp in to_add]) snapshots = [snp for snp in snapshots if snp.id in missing] diff --git a/swh/storage/filter.py b/swh/storage/filter.py --- a/swh/storage/filter.py +++ b/swh/storage/filter.py @@ -37,25 +37,25 @@ raise AttributeError(key) return getattr(self.storage, key) - def content_add(self, content: List[Content]) -> Dict: + def content_add(self, content: List[Content]) -> Dict[str, int]: contents_to_add = self._filter_missing_contents(content) return self.storage.content_add( [x for x in content if x.sha256 in contents_to_add] ) - def skipped_content_add(self, content: List[SkippedContent]) -> Dict: + def skipped_content_add(self, content: List[SkippedContent]) -> Dict[str, int]: contents_to_add = self._filter_missing_skipped_contents(content) return self.storage.skipped_content_add( [x for x in content if x.sha1_git is None or x.sha1_git in contents_to_add] ) - def directory_add(self, directories: List[Directory]) -> Dict: + def directory_add(self, directories: List[Directory]) -> Dict[str, int]: missing_ids = self._filter_missing_ids("directory", (d.id for d in directories)) return self.storage.directory_add( [d for d in directories if d.id in missing_ids] ) - def revision_add(self, revisions: List[Revision]) -> Dict: + def revision_add(self, revisions: List[Revision]) -> Dict[str, int]: missing_ids = self._filter_missing_ids("revision", (r.id for r in revisions)) return self.storage.revision_add([r for r in revisions if r.id in missing_ids]) diff --git a/swh/storage/interface.py b/swh/storage/interface.py --- a/swh/storage/interface.py +++ b/swh/storage/interface.py @@ -74,7 +74,7 @@ ... @remote_api_endpoint("content/add") - def content_add(self, content: List[Content]) -> Dict: + def content_add(self, content: List[Content]) -> Dict[str, int]: """Add content blobs to the storage Args: @@ -297,7 +297,7 @@ ... @remote_api_endpoint("content/skipped/add") - def skipped_content_add(self, content: List[SkippedContent]) -> Dict: + def skipped_content_add(self, content: List[SkippedContent]) -> Dict[str, int]: """Add contents to the skipped_content list, which contains (partial) information about content missing from the archive. @@ -350,7 +350,7 @@ ... @remote_api_endpoint("directory/add") - def directory_add(self, directories: List[Directory]) -> Dict: + def directory_add(self, directories: List[Directory]) -> Dict[str, int]: """Add directories to the storage Args: @@ -436,7 +436,7 @@ ... @remote_api_endpoint("revision/add") - def revision_add(self, revisions: List[Revision]) -> Dict: + def revision_add(self, revisions: List[Revision]) -> Dict[str, int]: """Add revisions to the storage Args: @@ -587,7 +587,7 @@ ... @remote_api_endpoint("release/add") - def release_add(self, releases: List[Release]) -> Dict: + def release_add(self, releases: List[Release]) -> Dict[str, int]: """Add releases to the storage Args: @@ -652,7 +652,7 @@ ... @remote_api_endpoint("snapshot/add") - def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: + def snapshot_add(self, snapshots: List[Snapshot]) -> Dict[str, int]: """Add snapshots to the storage. Args: diff --git a/swh/storage/postgresql/storage.py b/swh/storage/postgresql/storage.py --- a/swh/storage/postgresql/storage.py +++ b/swh/storage/postgresql/storage.py @@ -210,7 +210,7 @@ @timed @process_metrics - def content_add(self, content: List[Content]) -> Dict: + def content_add(self, content: List[Content]) -> Dict[str, int]: ctime = now() contents = [attr.evolve(c, ctime=ctime) for c in content] @@ -419,7 +419,7 @@ @db_transaction() def skipped_content_add( self, content: List[SkippedContent], db=None, cur=None - ) -> Dict: + ) -> Dict[str, int]: ctime = now() content = [attr.evolve(c, ctime=ctime) for c in content] @@ -457,7 +457,9 @@ @timed @process_metrics @db_transaction() - def directory_add(self, directories: List[Directory], db=None, cur=None) -> Dict: + def directory_add( + self, directories: List[Directory], db=None, cur=None + ) -> Dict[str, int]: summary = {"directory:add": 0} dirs = set() @@ -548,7 +550,9 @@ @timed @process_metrics @db_transaction() - def revision_add(self, revisions: List[Revision], db=None, cur=None) -> Dict: + def revision_add( + self, revisions: List[Revision], db=None, cur=None + ) -> Dict[str, int]: summary = {"revision:add": 0} revisions_missing = set( @@ -687,7 +691,7 @@ @timed @process_metrics @db_transaction() - def release_add(self, releases: List[Release], db=None, cur=None) -> Dict: + def release_add(self, releases: List[Release], db=None, cur=None) -> Dict[str, int]: summary = {"release:add": 0} release_ids = set(release.id for release in releases) @@ -743,7 +747,9 @@ @timed @process_metrics @db_transaction() - def snapshot_add(self, snapshots: List[Snapshot], db=None, cur=None) -> Dict: + def snapshot_add( + self, snapshots: List[Snapshot], db=None, cur=None + ) -> Dict[str, int]: created_temp_table = False count = 0 diff --git a/swh/storage/validate.py b/swh/storage/validate.py --- a/swh/storage/validate.py +++ b/swh/storage/validate.py @@ -45,7 +45,7 @@ f"but it should be {hash_to_hex(id_)}: {obj}" ) - def content_add(self, content: List[Content]) -> Dict: + def content_add(self, content: List[Content]) -> Dict[str, int]: for cont in content: hashes = MultiHash.from_data(cont.data).digest() if hashes != cont.hashes(): @@ -54,18 +54,18 @@ ) return self.storage.content_add(content) - def directory_add(self, directories: List[Directory]) -> Dict: + def directory_add(self, directories: List[Directory]) -> Dict[str, int]: self._check_hashes(directories) return self.storage.directory_add(directories) - def revision_add(self, revisions: List[Revision]) -> Dict: + def revision_add(self, revisions: List[Revision]) -> Dict[str, int]: self._check_hashes(revisions) return self.storage.revision_add(revisions) - def release_add(self, releases: List[Release]) -> Dict: + def release_add(self, releases: List[Release]) -> Dict[str, int]: self._check_hashes(releases) return self.storage.release_add(releases) - def snapshot_add(self, snapshots: List[Snapshot]) -> Dict: + def snapshot_add(self, snapshots: List[Snapshot]) -> Dict[str, int]: self._check_hashes(snapshots) return self.storage.snapshot_add(snapshots)