Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 67 Lines • ▼ Show 20 Lines | def reset(self): | ||||
| self._sorted_sha1s = [] | self._sorted_sha1s = [] | ||||
| self.objstorage = get_objstorage('memory', {}) | self.objstorage = get_objstorage('memory', {}) | ||||
| def check_config(self, *, check_write): | def check_config(self, *, check_write): | ||||
| """Check that the storage is configured and ready to go.""" | """Check that the storage is configured and ready to go.""" | ||||
| return True | return True | ||||
| def _content_add(self, contents, with_data): | def _content_add(self, contents, with_data): | ||||
olasd: the argument should probably be renamed `content_and_origins` | |||||
| if self.journal_writer: | if self.journal_writer: | ||||
| for content in contents: | for content in contents: | ||||
| content = attr.evolve(content, data=None) | content = attr.evolve(content, data=None) | ||||
| self.journal_writer.write_addition('content', content) | self.journal_writer.write_addition('content', content) | ||||
| content_with_data = [] | content_with_data = [] | ||||
| content_without_data = [] | content_without_data = [] | ||||
| for content in contents: | for content in contents: | ||||
| ▲ Show 20 Lines • Show All 47 Lines • ▼ Show 20 Lines | def _content_add_present(self, contents, with_data): | ||||
| if with_data: | if with_data: | ||||
| content_data = self._contents[key].data | content_data = self._contents[key].data | ||||
| self._contents[key].data = None | self._contents[key].data = None | ||||
| count_content_bytes_added += len(content_data) | count_content_bytes_added += len(content_data) | ||||
| self.objstorage.add(content_data, content.sha1) | self.objstorage.add(content_data, content.sha1) | ||||
| return (count_content_added, count_content_bytes_added) | return (count_content_added, count_content_bytes_added) | ||||
| def _content_add_absent(self, contents): | def _content_add_absent(self, contents): | ||||
Not Done Inline Actionscontents_and_origins :P olasd: `contents_and_origins` :P | |||||
| count = 0 | count = 0 | ||||
| skipped_content_missing = self.skipped_content_missing(contents) | skipped_content_missing = self.skipped_content_missing(contents) | ||||
| for content in skipped_content_missing: | for content in skipped_content_missing: | ||||
Not Done Inline ActionsSurely that only works because we only add contents from a single origin at a time; after the filtering, skipped_content_missing and origins aren't the same length any more. You really need to pass the full content to skipped_content_missing, then do the content/origin splitting. Which, in addition to the double-zipping, makes me wonder if that's really the right way to go at all. olasd: Surely that only works because we only add contents from a single origin at a time; after the… | |||||
| key = self._content_key(content) | key = self._content_key(content) | ||||
| for algo in DEFAULT_ALGORITHMS: | for algo in DEFAULT_ALGORITHMS: | ||||
| self._skipped_content_indexes[algo][content.get_hash(algo)] \ | self._skipped_content_indexes[algo][content.get_hash(algo)] \ | ||||
| .add(key) | .add(key) | ||||
| self._skipped_contents[key] = content | self._skipped_contents[key] = content | ||||
| count += 1 | count += 1 | ||||
| return count | return count | ||||
| def _content_to_model(self, contents): | |||||
| """Takes a list of content dicts, optionally with an extra 'origin' | |||||
| key, and yields tuples (model.Content, origin).""" | |||||
| for content in contents: | |||||
| content = content.copy() | |||||
| content.pop('origin', None) | |||||
Not Done Inline ActionsCould you turn this into a for loop? This isn't very readable. olasd: Could you turn this into a for loop? This isn't very readable. | |||||
| yield Content.from_dict(content) | |||||
| def content_add(self, content): | def content_add(self, content): | ||||
| """Add content blobs to the storage | """Add content blobs to the storage | ||||
| Args: | Args: | ||||
| content (iterable): iterable of dictionaries representing | content (iterable): iterable of dictionaries representing | ||||
| individual pieces of content to add. Each dictionary has the | individual pieces of content to add. Each dictionary has the | ||||
| following keys: | following keys: | ||||
| Show All 13 Lines | def content_add(self, content): | ||||
| Returns: | Returns: | ||||
| Summary dict with the following key and associated values: | Summary dict with the following key and associated values: | ||||
| content:add: New contents added | content:add: New contents added | ||||
| content_bytes:add: Sum of the contents' length data | content_bytes:add: Sum of the contents' length data | ||||
| skipped_content:add: New skipped contents (no data) added | skipped_content:add: New skipped contents (no data) added | ||||
| """ | """ | ||||
| content = [Content.from_dict(c) for c in content] | content = list(self._content_to_model(content)) | ||||
Not Done Inline Actionscontent_and_origins? olasd: `content_and_origins`? | |||||
| now = datetime.datetime.now(tz=datetime.timezone.utc) | now = datetime.datetime.now(tz=datetime.timezone.utc) | ||||
| for item in content: | for item in content: | ||||
| item.ctime = now | item.ctime = now | ||||
| return self._content_add(content, with_data=True) | return self._content_add(content, with_data=True) | ||||
| def content_add_metadata(self, content): | def content_add_metadata(self, content): | ||||
| """Add content metadata to the storage (like `content_add`, but | """Add content metadata to the storage (like `content_add`, but | ||||
| without inserting to the objstorage). | without inserting to the objstorage). | ||||
| Show All 18 Lines | def content_add_metadata(self, content): | ||||
| Returns: | Returns: | ||||
| Summary dict with the following key and associated values: | Summary dict with the following key and associated values: | ||||
| content:add: New contents added | content:add: New contents added | ||||
| skipped_content:add: New skipped contents (no data) added | skipped_content:add: New skipped contents (no data) added | ||||
| """ | """ | ||||
| content = [Content.from_dict(c) for c in content] | content = list(self._content_to_model(content)) | ||||
| return self._content_add(content, with_data=False) | return self._content_add(content, with_data=False) | ||||
| def content_get(self, content): | def content_get(self, content): | ||||
| """Retrieve in bulk contents and their data. | """Retrieve in bulk contents and their data. | ||||
| This function may yield more blobs than provided sha1 identifiers, | This function may yield more blobs than provided sha1 identifiers, | ||||
| in case they collide. | in case they collide. | ||||
| ▲ Show 20 Lines • Show All 1,502 Lines • Show Last 20 Lines | |||||
the argument should probably be renamed content_and_origins