Differential D3692 Diff 13008 swh/storage/in_memory.py

Changeset View

Standalone View

swh/storage/in_memory.py

Show All 18 Lines	from typing import (
Callable,		Callable,
Dict,		Dict,
Generic,		Generic,
Hashable,		Hashable,
Iterable,		Iterable,
Iterator,		Iterator,
List,		List,
Optional,		Optional,
		Sequence,
Tuple,		Tuple,
TypeVar,		TypeVar,
Union,		Union,
)		)

import attr		import attr

from swh.core.api.serializers import msgpack_loads, msgpack_dumps		from swh.core.api.serializers import msgpack_loads, msgpack_dumps
▲ Show 20 Lines • Show All 281 Lines • ▼ Show 20 Lines	def content_get_metadata(self, contents: List[bytes]) -> Dict[bytes, List[Dict]]:
for key in objs:		for key in objs:
d = self._contents[key].to_dict()		d = self._contents[key].to_dict()
del d["ctime"]		del d["ctime"]
if "data" in d:		if "data" in d:
del d["data"]		del d["data"]
result[sha1].append(d)		result[sha1].append(d)
return result		return result

def content_find(self, content):		def content_find(self, content: Dict[str, Any]) -> Sequence[Content]:
if not set(content).intersection(DEFAULT_ALGORITHMS):		if not set(content).intersection(DEFAULT_ALGORITHMS):
raise StorageArgumentException(		raise StorageArgumentException(
"content keys must contain at least one of: %s"		"content keys must contain at least one "
		ardumontAuthorUnsubmitted Done Inline Actions same alignment with f-strings we starting using. ardumont: same alignment with f-strings we starting using.
% ", ".join(sorted(DEFAULT_ALGORITHMS))		f"of: {', '.join(sorted(DEFAULT_ALGORITHMS))}"
)		)
found = []		found = []
for algo in DEFAULT_ALGORITHMS:		for algo in DEFAULT_ALGORITHMS:
hash = content.get(algo)		hash = content.get(algo)
if hash and hash in self._content_indexes[algo]:		if hash and hash in self._content_indexes[algo]:
found.append(self._content_indexes[algo][hash])		found.append(self._content_indexes[algo][hash])

if not found:		if not found:
return []		return []

keys = list(set.intersection(*found))		keys = list(set.intersection(*found))
return [self._contents[key].to_dict() for key in keys]		return [self._contents[key] for key in keys]

def content_missing(self, content, key_hash="sha1"):		def content_missing(self, content, key_hash="sha1"):
for cont in content:		for cont in content:
for (algo, hash_) in cont.items():		for (algo, hash_) in cont.items():
if algo not in DEFAULT_ALGORITHMS:		if algo not in DEFAULT_ALGORITHMS:
continue		continue
if hash_ not in self._content_indexes.get(algo, []):		if hash_ not in self._content_indexes.get(algo, []):
yield cont[key_hash]		yield cont[key_hash]
break		break
else:
for result in self.content_find(cont):
if result["status"] == "missing":
yield cont[key_hash]
ardumontAuthorUnsubmitted Done Inline Actions impossible path mentioned in description. ardumont: impossible path mentioned in description.

def content_missing_per_sha1(self, contents):		def content_missing_per_sha1(self, contents):
for content in contents:		for content in contents:
if content not in self._content_indexes["sha1"]:		if content not in self._content_indexes["sha1"]:
yield content		yield content

def content_missing_per_sha1_git(self, contents):		def content_missing_per_sha1_git(self, contents):
for content in contents:		for content in contents:
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines	def directory_add(self, directories: List[Directory]) -> Dict:

return {"directory:add": count}		return {"directory:add": count}

def directory_missing(self, directories):		def directory_missing(self, directories):
for id in directories:		for id in directories:
if id not in self._directories:		if id not in self._directories:
yield id		yield id

def _join_dentry_to_content(self, dentry):		def _join_dentry_to_content(self, dentry: Dict[str, Any]) -> Dict[str, Any]:
keys = (		keys = (
"status",		"status",
"sha1",		"sha1",
"sha1_git",		"sha1_git",
"sha256",		"sha256",
"length",		"length",
)		)
ret = dict.fromkeys(keys)		ret = dict.fromkeys(keys)
ret.update(dentry)		ret.update(dentry)
if ret["type"] == "file":		if ret["type"] == "file":
# TODO: Make it able to handle more than one content		# TODO: Make it able to handle more than one content
content = self.content_find({"sha1_git": ret["target"]})		contents = self.content_find({"sha1_git": ret["target"]})
if content:		if contents:
		ardumontAuthorUnsubmitted Done Inline Actions D3694 ardumont: D3694
content = content[0]		content = contents[0]
ardumontAuthorUnsubmitted Done Inline Actions Typoed, uncaught because we don't have coverage here, nice! getattr(content, key). And why did not mypy complain about `swh/storage/in_memory.py:431: error: Value of type "Iterable[Content]" is not indexable` earlier? ardumont: Typoed, uncaught because we don't have coverage here, nice! ``` getattr(content, key). ```…
ardumontAuthorUnsubmitted Done Inline Actions And why did not mypy complain ... I gather mypy's check heuristic is basically skipping untyped methods (or something). I have the error now that i tried to type this currently untyped method. ardumont: > And why did not mypy complain ... I gather mypy's check heuristic is basically skipping…
ardumontAuthorUnsubmitted Done Inline Actions (webapp tests caught it though) ardumont: (webapp tests caught it though)
ardumontAuthorUnsubmitted Done Inline Actions yeah, ok, the tests testing the `directory-ls` function only create the directories and not the contents targetted by the directories so we never pass here. I'll fix it in another diff. ardumont: yeah, ok, the tests testing the `directory-ls` function only create the directories and not the…
ardumontAuthorUnsubmitted Done Inline Actions D3694 ardumont: D3694
for key in keys:		for key in keys:
ret[key] = content[key]		ret[key] = getattr(content, key)
return ret		return ret

def _directory_ls(self, directory_id, recursive, prefix=b""):		def _directory_ls(self, directory_id, recursive, prefix=b""):
if directory_id in self._directories:		if directory_id in self._directories:
for entry in self._directories[directory_id].entries:		for entry in self._directories[directory_id].entries:
ret = self._join_dentry_to_content(entry.to_dict())		ret = self._join_dentry_to_content(entry.to_dict())
ret["name"] = prefix + ret["name"]		ret["name"] = prefix + ret["name"]
ret["dir_id"] = directory_id		ret["dir_id"] = directory_id
▲ Show 20 Lines • Show All 822 Lines • Show Last 20 Lines