Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/in_memory.py
| Show First 20 Lines • Show All 1,240 Lines • ▼ Show 20 Lines | def origin_visit_add(self, origin, date, type): | ||||
| 'origin': origin.url, | 'origin': origin.url, | ||||
| 'visit': visit_id, | 'visit': visit_id, | ||||
| } | } | ||||
| self._objects[(origin_url, visit_id)].append( | self._objects[(origin_url, visit_id)].append( | ||||
| ('origin_visit', None)) | ('origin_visit', None)) | ||||
| if self.journal_writer: | if self.journal_writer: | ||||
| visit = attr.evolve(visit, origin=origin) | |||||
| self.journal_writer.write_addition('origin_visit', visit) | self.journal_writer.write_addition('origin_visit', visit) | ||||
| return visit_ret | return visit_ret | ||||
| def origin_visit_update(self, origin, visit_id, status=None, | def origin_visit_update(self, origin, visit_id, status=None, | ||||
| metadata=None, snapshot=None): | metadata=None, snapshot=None): | ||||
| """Update an origin_visit's status. | """Update an origin_visit's status. | ||||
| Args: | Args: | ||||
| origin (str): visited origin's URL | origin (str): visited origin's URL | ||||
| visit_id (int): visit's identifier | visit_id (int): visit's identifier | ||||
| status: visit's new status | status: visit's new status | ||||
| metadata: data associated to the visit | metadata: data associated to the visit | ||||
| snapshot (sha1_git): identifier of the snapshot to add to | snapshot (sha1_git): identifier of the snapshot to add to | ||||
| the visit | the visit | ||||
| Returns: | Returns: | ||||
| None | None | ||||
| """ | """ | ||||
| if not isinstance(origin, str): | |||||
| raise TypeError('origin must be a string, not %r' % (origin,)) | |||||
| origin_url = self._get_origin_url(origin) | origin_url = self._get_origin_url(origin) | ||||
| if origin_url is None: | if origin_url is None: | ||||
| raise ValueError('Unknown origin.') | raise ValueError('Unknown origin.') | ||||
| try: | try: | ||||
| visit = self._origin_visits[origin_url][visit_id-1] | visit = self._origin_visits[origin_url][visit_id-1] | ||||
| except IndexError: | except IndexError: | ||||
| raise ValueError('Unknown visit_id for this origin') \ | raise ValueError('Unknown visit_id for this origin') \ | ||||
| from None | from None | ||||
| updates = {} | updates = {} | ||||
| if status: | if status: | ||||
| updates['status'] = status | updates['status'] = status | ||||
| if metadata: | if metadata: | ||||
| updates['metadata'] = metadata | updates['metadata'] = metadata | ||||
| if snapshot: | if snapshot: | ||||
| updates['snapshot'] = snapshot | updates['snapshot'] = snapshot | ||||
| visit = attr.evolve(visit, **updates) | visit = attr.evolve(visit, **updates) | ||||
| if self.journal_writer: | if self.journal_writer: | ||||
| origin = self._origins[origin_url] | self.journal_writer.write_update('origin_visit', visit) | ||||
| journal_visit = attr.evolve(visit, origin=origin) | |||||
| self.journal_writer.write_update('origin_visit', journal_visit) | |||||
| self._origin_visits[origin_url][visit_id-1] = visit | self._origin_visits[origin_url][visit_id-1] = visit | ||||
| if origin_url not in self._origin_visits or \ | |||||
| visit_id > len(self._origin_visits[origin_url]): | |||||
| return | |||||
| def origin_visit_upsert(self, visits): | def origin_visit_upsert(self, visits): | ||||
| """Add a origin_visits with a specific id and with all its data. | """Add a origin_visits with a specific id and with all its data. | ||||
| If there is already an origin_visit with the same | If there is already an origin_visit with the same | ||||
| `(origin_url, visit_id)`, updates it instead of inserting a new one. | `(origin_url, visit_id)`, updates it instead of inserting a new one. | ||||
| Args: | Args: | ||||
| visits: iterable of dicts with keys: | visits: iterable of dicts with keys: | ||||
| origin: dict with keys either `id` or `url` | origin: origin url | ||||
| visit: origin visit id | visit: origin visit id | ||||
| type: type of loader used for the visit | type: type of loader used for the visit | ||||
| date: timestamp of such visit | date: timestamp of such visit | ||||
| status: Visit's new status | status: Visit's new status | ||||
| metadata: Data associated to the visit | metadata: Data associated to the visit | ||||
| snapshot (sha1_git): identifier of the snapshot to add to | snapshot (sha1_git): identifier of the snapshot to add to | ||||
| the visit | the visit | ||||
| """ | """ | ||||
| for visit in visits: | |||||
| if not isinstance(visit['origin'], str): | |||||
| raise TypeError("visit['origin'] must be a string, not %r" | |||||
| % (visit['origin'],)) | |||||
| visits = [OriginVisit.from_dict(d) for d in visits] | visits = [OriginVisit.from_dict(d) for d in visits] | ||||
| if self.journal_writer: | if self.journal_writer: | ||||
| for visit in visits: | for visit in visits: | ||||
| visit = attr.evolve( | |||||
| visit, | |||||
| origin=self._origins[visit.origin.url]) | |||||
| self.journal_writer.write_addition('origin_visit', visit) | self.journal_writer.write_addition('origin_visit', visit) | ||||
| for visit in visits: | for visit in visits: | ||||
| visit_id = visit.visit | visit_id = visit.visit | ||||
| origin_url = visit.origin.url | origin_url = visit.origin | ||||
| visit = attr.evolve(visit, origin=origin_url) | visit = attr.evolve(visit, origin=origin_url) | ||||
| self._objects[(origin_url, visit_id)].append( | self._objects[(origin_url, visit_id)].append( | ||||
| ('origin_visit', None)) | ('origin_visit', None)) | ||||
| while len(self._origin_visits[origin_url]) <= visit_id: | while len(self._origin_visits[origin_url]) <= visit_id: | ||||
| self._origin_visits[origin_url].append(None) | self._origin_visits[origin_url].append(None) | ||||
| ▲ Show 20 Lines • Show All 351 Lines • Show Last 20 Lines | |||||