Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
Show First 20 Lines • Show All 315 Lines • ▼ Show 20 Lines | def content_get_metadata(self, contents: List[bytes]) -> Dict[bytes, List[Dict]]: | ||||
for key in objs: | for key in objs: | ||||
d = self._contents[key].to_dict() | d = self._contents[key].to_dict() | ||||
del d["ctime"] | del d["ctime"] | ||||
if "data" in d: | if "data" in d: | ||||
del d["data"] | del d["data"] | ||||
result[sha1].append(d) | result[sha1].append(d) | ||||
return result | return result | ||||
def content_find(self, content): | def content_find(self, content: Dict[str, Any]) -> List[Content]: | ||||
if not set(content).intersection(DEFAULT_ALGORITHMS): | if not set(content).intersection(DEFAULT_ALGORITHMS): | ||||
raise StorageArgumentException( | raise StorageArgumentException( | ||||
"content keys must contain at least one of: %s" | "content keys must contain at least one " | ||||
ardumont: same alignment with f-strings we starting using. | |||||
% ", ".join(sorted(DEFAULT_ALGORITHMS)) | f"of: {', '.join(sorted(DEFAULT_ALGORITHMS))}" | ||||
) | ) | ||||
found = [] | found = [] | ||||
for algo in DEFAULT_ALGORITHMS: | for algo in DEFAULT_ALGORITHMS: | ||||
hash = content.get(algo) | hash = content.get(algo) | ||||
if hash and hash in self._content_indexes[algo]: | if hash and hash in self._content_indexes[algo]: | ||||
found.append(self._content_indexes[algo][hash]) | found.append(self._content_indexes[algo][hash]) | ||||
if not found: | if not found: | ||||
return [] | return [] | ||||
keys = list(set.intersection(*found)) | keys = list(set.intersection(*found)) | ||||
return [self._contents[key].to_dict() for key in keys] | return [self._contents[key] for key in keys] | ||||
def content_missing(self, content, key_hash="sha1"): | def content_missing(self, content, key_hash="sha1"): | ||||
for cont in content: | for cont in content: | ||||
for (algo, hash_) in cont.items(): | for (algo, hash_) in cont.items(): | ||||
if algo not in DEFAULT_ALGORITHMS: | if algo not in DEFAULT_ALGORITHMS: | ||||
continue | continue | ||||
if hash_ not in self._content_indexes.get(algo, []): | if hash_ not in self._content_indexes.get(algo, []): | ||||
yield cont[key_hash] | yield cont[key_hash] | ||||
break | break | ||||
else: | |||||
for result in self.content_find(cont): | |||||
if result["status"] == "missing": | |||||
yield cont[key_hash] | |||||
Done Inline Actionsimpossible path mentioned in description. ardumont: impossible path mentioned in description. | |||||
def content_missing_per_sha1(self, contents): | def content_missing_per_sha1(self, contents): | ||||
for content in contents: | for content in contents: | ||||
if content not in self._content_indexes["sha1"]: | if content not in self._content_indexes["sha1"]: | ||||
yield content | yield content | ||||
def content_missing_per_sha1_git(self, contents): | def content_missing_per_sha1_git(self, contents): | ||||
for content in contents: | for content in contents: | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | def directory_add(self, directories: List[Directory]) -> Dict: | ||||
return {"directory:add": count} | return {"directory:add": count} | ||||
def directory_missing(self, directories): | def directory_missing(self, directories): | ||||
for id in directories: | for id in directories: | ||||
if id not in self._directories: | if id not in self._directories: | ||||
yield id | yield id | ||||
def _join_dentry_to_content(self, dentry): | def _join_dentry_to_content(self, dentry: Dict[str, Any]) -> Dict[str, Any]: | ||||
keys = ( | keys = ( | ||||
"status", | "status", | ||||
"sha1", | "sha1", | ||||
"sha1_git", | "sha1_git", | ||||
"sha256", | "sha256", | ||||
"length", | "length", | ||||
) | ) | ||||
ret = dict.fromkeys(keys) | ret = dict.fromkeys(keys) | ||||
ret.update(dentry) | ret.update(dentry) | ||||
if ret["type"] == "file": | if ret["type"] == "file": | ||||
# TODO: Make it able to handle more than one content | # TODO: Make it able to handle more than one content | ||||
content = self.content_find({"sha1_git": ret["target"]}) | contents = self.content_find({"sha1_git": ret["target"]}) | ||||
if content: | if contents: | ||||
Done Inline Actionsardumont: D3694 | |||||
content = content[0] | content = contents[0] | ||||
Done Inline ActionsTypoed, uncaught because we don't have coverage here, nice! getattr(content, key). And why did not mypy complain about swh/storage/in_memory.py:431: error: Value of type "Iterable[Content]" is not indexable earlier? ardumont: Typoed, uncaught because we don't have coverage here, nice!
```
getattr(content, key).
```… | |||||
Done Inline Actions
I gather mypy's check heuristic is basically skipping untyped methods (or something). ardumont: > And why did not mypy complain ...
I gather mypy's check heuristic is basically skipping… | |||||
Done Inline Actions(webapp tests caught it though) ardumont: (webapp tests caught it though) | |||||
Done Inline Actionsyeah, ok, the tests testing the directory-ls function only create the directories and not the contents targetted by the directories so we never pass here. I'll fix it in another diff. ardumont: yeah, ok, the tests testing the `directory-ls` function only create the directories and not the… | |||||
Done Inline Actionsardumont: D3694 | |||||
for key in keys: | for key in keys: | ||||
ret[key] = content[key] | ret[key] = getattr(content, key) | ||||
return ret | return ret | ||||
def _directory_ls(self, directory_id, recursive, prefix=b""): | def _directory_ls(self, directory_id, recursive, prefix=b""): | ||||
if directory_id in self._directories: | if directory_id in self._directories: | ||||
for entry in self._directories[directory_id].entries: | for entry in self._directories[directory_id].entries: | ||||
ret = self._join_dentry_to_content(entry.to_dict()) | ret = self._join_dentry_to_content(entry.to_dict()) | ||||
ret["name"] = prefix + ret["name"] | ret["name"] = prefix + ret["name"] | ||||
ret["dir_id"] = directory_id | ret["dir_id"] = directory_id | ||||
▲ Show 20 Lines • Show All 822 Lines • Show Last 20 Lines |
same alignment with f-strings we starting using.