Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show First 20 Lines • Show All 834 Lines • ▼ Show 20 Lines | ) -> OriginVisit: | ||||
| metadata=None, | metadata=None, | ||||
| ) | ) | ||||
| self._origin_visit_status_add(visit_status) | self._origin_visit_status_add(visit_status) | ||||
| return visit | return visit | ||||
| def _origin_visit_status_add(self, visit_status: OriginVisitStatus) -> None: | def _origin_visit_status_add(self, visit_status: OriginVisitStatus) -> None: | ||||
| """Add an origin visit status""" | """Add an origin visit status""" | ||||
| self.journal_writer.origin_visit_status_add([visit_status]) | |||||
| self._cql_runner.origin_visit_status_add_one(visit_status) | self._cql_runner.origin_visit_status_add_one(visit_status) | ||||
| def origin_visit_status_add( | def origin_visit_status_add( | ||||
| self, visit_statuses: Iterable[OriginVisitStatus] | self, visit_statuses: Iterable[OriginVisitStatus] | ||||
| ) -> None: | ) -> None: | ||||
| # First round to check existence (fail early if any is ko) | # First round to check existence (fail early if any is ko) | ||||
| for visit_status in visit_statuses: | for visit_status in visit_statuses: | ||||
| origin_url = self.origin_get({"url": visit_status.origin}) | origin_url = self.origin_get({"url": visit_status.origin}) | ||||
| Show All 40 Lines | ): | ||||
| visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
| origin=origin_url, | origin=origin_url, | ||||
| visit=visit_id, | visit=visit_id, | ||||
| date=date or now(), | date=date or now(), | ||||
| status=status, | status=status, | ||||
| snapshot=snapshot or last_visit_update["snapshot"], | snapshot=snapshot or last_visit_update["snapshot"], | ||||
| metadata=metadata or last_visit_update["metadata"], | metadata=metadata or last_visit_update["metadata"], | ||||
| ) | ) | ||||
| self._origin_visit_status_add(visit_status) | self._cql_runner.origin_visit_status_add_one(visit_status) | ||||
| # self._origin_visit_status_add(visit_status) | |||||
vlorentz: why this change? | |||||
Done Inline ActionsIt's only a temporary change. Now that self._origin_visit_status_add writes to the journal, the journal testing part is a mess of origin-visit and origin-visit status with differing date... I want to concern myself here only with origin-visit-add without the harassment i had so far with the update part. If you want I can add that as a FIXME. ardumont: It's only a temporary change.
Now that self._origin_visit_status_add writes to the journal… | |||||
Done Inline Actions
D3244 ;) ardumont: > So i'll deal with that in another diff which will revert that behavior (thus why it's… | |||||
| def _origin_visit_merge( | def _origin_visit_merge( | ||||
| self, visit: Dict[str, Any], visit_status: Dict[str, Any] | self, visit: Dict[str, Any], visit_status: Dict[str, Any] | ||||
| ) -> Dict[str, Any]: | ) -> Dict[str, Any]: | ||||
| """Merge origin_visit and visit_status together. | """Merge origin_visit and visit_status together. | ||||
| """ | """ | ||||
| return OriginVisit.from_dict( | return OriginVisit.from_dict( | ||||
| { | { | ||||
| ▲ Show 20 Lines • Show All 43 Lines • ▼ Show 20 Lines | def origin_visit_upsert(self, visits: Iterable[OriginVisit]) -> None: | ||||
| visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
| origin=visit.origin, | origin=visit.origin, | ||||
| visit=visit.visit, | visit=visit.visit, | ||||
| date=now(), | date=now(), | ||||
| status=visit.status, | status=visit.status, | ||||
| snapshot=visit.snapshot, | snapshot=visit.snapshot, | ||||
| metadata=visit.metadata, | metadata=visit.metadata, | ||||
| ) | ) | ||||
| self._origin_visit_status_add(visit_status) | self._cql_runner.origin_visit_status_add_one(visit_status) | ||||
| @staticmethod | @staticmethod | ||||
| def _format_origin_visit_row(visit): | def _format_origin_visit_row(visit): | ||||
| return { | return { | ||||
| **visit._asdict(), | **visit._asdict(), | ||||
| "origin": visit.origin, | "origin": visit.origin, | ||||
| "date": visit.date.replace(tzinfo=datetime.timezone.utc), | "date": visit.date.replace(tzinfo=datetime.timezone.utc), | ||||
| "metadata": (json.loads(visit.metadata) if visit.metadata else None), | "metadata": (json.loads(visit.metadata) if visit.metadata else None), | ||||
| ▲ Show 20 Lines • Show All 251 Lines • Show Last 20 Lines | |||||
why this change?