Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 68 Lines • ▼ Show 20 Lines | def _content_add(self, contents, with_data): | ||||
| count_contents = 0 | count_contents = 0 | ||||
| count_content_added = 0 | count_content_added = 0 | ||||
| count_content_bytes_added = 0 | count_content_bytes_added = 0 | ||||
| for content in contents: | for content in contents: | ||||
| key = self._content_key(content) | key = self._content_key(content) | ||||
| if key in self._contents: | if key in self._contents: | ||||
| self._objects[key].append(('skipped_content', content['sha1'])) | |||||
| continue | continue | ||||
| for algorithm in DEFAULT_ALGORITHMS: | for algorithm in DEFAULT_ALGORITHMS: | ||||
| if content[algorithm] in self._content_indexes[algorithm]: | if content[algorithm] in self._content_indexes[algorithm]: | ||||
| from . import HashCollision | from . import HashCollision | ||||
| raise HashCollision(algorithm, content[algorithm], key) | raise HashCollision(algorithm, content[algorithm], key) | ||||
| for algorithm in DEFAULT_ALGORITHMS: | for algorithm in DEFAULT_ALGORITHMS: | ||||
| self._content_indexes[algorithm][content[algorithm]].add(key) | self._content_indexes[algorithm][content[algorithm]].add(key) | ||||
| self._objects[content['sha1_git']].append( | self._objects[content['sha1_git']].append( | ||||
| ▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines | def content_add_metadata(self, contents): | ||||
| Summary dict with the following key and associated values: | Summary dict with the following key and associated values: | ||||
| content:add: New contents added | content:add: New contents added | ||||
| skipped_content:add: New skipped contents (no data) added | skipped_content:add: New skipped contents (no data) added | ||||
| """ | """ | ||||
| return self._content_add(contents, with_data=False) | return self._content_add(contents, with_data=False) | ||||
| def content_get(self, ids): | def content_get(self, ids, start=0, end=-1): | ||||
| """Retrieve in bulk contents and their data. | """Retrieve in bulk contents and their data. | ||||
| This function may yield more blobs than provided sha1 identifiers, | This function may yield more blobs than provided sha1 identifiers, | ||||
| in case they collide. | in case they collide. | ||||
| Args: | Args: | ||||
| content: iterables of sha1 | content: iterables of sha1 | ||||
| ▲ Show 20 Lines • Show All 984 Lines • ▼ Show 20 Lines | def origin_visit_add(self, origin, date=None, *, ts=None): | ||||
| 'metadata': None, | 'metadata': None, | ||||
| 'visit': visit_id | 'visit': visit_id | ||||
| } | } | ||||
| self._origin_visits[origin_id-1].append(visit) | self._origin_visits[origin_id-1].append(visit) | ||||
| visit_ret = { | visit_ret = { | ||||
| 'origin': origin_id, | 'origin': origin_id, | ||||
| 'visit': visit_id, | 'visit': visit_id, | ||||
| } | } | ||||
| key = (origin_id, visit_id) | |||||
| self._objects[key].append(('origin_visit', origin_id)) | |||||
| if self.journal_writer: | if self.journal_writer: | ||||
| origin = self.origin_get([{'id': origin_id}])[0] | origin = self.origin_get([{'id': origin_id}])[0] | ||||
| del origin['id'] | del origin['id'] | ||||
| self.journal_writer.write_addition('origin_visit', { | self.journal_writer.write_addition('origin_visit', { | ||||
| **visit, 'origin': origin}) | **visit, 'origin': origin}) | ||||
| return visit_ret | return visit_ret | ||||
| ▲ Show 20 Lines • Show All 327 Lines • ▼ Show 20 Lines | def _person_add(self, person): | ||||
| Args: | Args: | ||||
| person: dictionary with keys fullname, name and email. | person: dictionary with keys fullname, name and email. | ||||
| """ | """ | ||||
| key = ('person', person['fullname']) | key = ('person', person['fullname']) | ||||
| if key not in self._objects: | if key not in self._objects: | ||||
| person_id = len(self._persons) + 1 | person_id = len(self._persons) + 1 | ||||
| self._persons.append(dict(person)) | self._persons.append(dict(person)) | ||||
| self._objects[key].append(('person', person_id)) | |||||
| else: | else: | ||||
| person_id = self._objects[key][0][1] | person_id = self._objects[key][0][1] | ||||
| p = next(self.person_get([person_id])) | p = next(self.person_get([person_id])) | ||||
| person.update(p.items()) | person.update(p.items()) | ||||
| person['id'] = person_id | person['id'] = person_id | ||||
| self._objects[key].append(('person', person_id)) | |||||
| @staticmethod | @staticmethod | ||||
| def _content_key(content): | def _content_key(content): | ||||
| """A stable key for a content""" | """A stable key for a content""" | ||||
| return tuple(content.get(key) for key in sorted(DEFAULT_ALGORITHMS)) | return tuple(content.get(key) for key in sorted(DEFAULT_ALGORITHMS)) | ||||
| @staticmethod | @staticmethod | ||||
| def _tool_key(tool): | def _tool_key(tool): | ||||
| return (tool['name'], tool['version'], | return (tool['name'], tool['version'], | ||||
| tuple(sorted(tool['configuration'].items()))) | tuple(sorted(tool['configuration'].items()))) | ||||
| @staticmethod | @staticmethod | ||||
| def _metadata_provider_key(provider): | def _metadata_provider_key(provider): | ||||
| return (provider['name'], provider['url']) | return (provider['name'], provider['url']) | ||||