Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/postgresql/storage.py
| Show First 20 Lines • Show All 838 Lines • ▼ Show 20 Lines | ) -> Iterable[OriginVisit]: | ||||
| assert visit.visit is not None | assert visit.visit is not None | ||||
| all_visits.append(visit) | all_visits.append(visit) | ||||
| # Forced to write after for the case when the visit has no id | # Forced to write after for the case when the visit has no id | ||||
| self.journal_writer.origin_visit_add([visit]) | self.journal_writer.origin_visit_add([visit]) | ||||
| visit_status = OriginVisitStatus( | visit_status = OriginVisitStatus( | ||||
| origin=visit.origin, | origin=visit.origin, | ||||
| visit=visit.visit, | visit=visit.visit, | ||||
| date=visit.date, | date=visit.date, | ||||
| type=visit.type, | |||||
| status="created", | status="created", | ||||
| snapshot=None, | snapshot=None, | ||||
| ) | ) | ||||
| self._origin_visit_status_add(visit_status, db=db, cur=cur) | self._origin_visit_status_add(visit_status, db=db, cur=cur) | ||||
| send_metric("origin_visit:add", count=nb_visits, method_name="origin_visit") | send_metric("origin_visit:add", count=nb_visits, method_name="origin_visit") | ||||
| return all_visits | return all_visits | ||||
| def _origin_visit_status_add( | def _origin_visit_status_add( | ||||
| self, visit_status: OriginVisitStatus, db, cur | self, visit_status: OriginVisitStatus, db, cur | ||||
| ) -> None: | ) -> None: | ||||
| """Add an origin visit status""" | """Add an origin visit status""" | ||||
| self.journal_writer.origin_visit_status_add([visit_status]) | self.journal_writer.origin_visit_status_add([visit_status]) | ||||
| db.origin_visit_status_add(visit_status, cur=cur) | db.origin_visit_status_add(visit_status, cur=cur) | ||||
| send_metric( | send_metric( | ||||
| "origin_visit_status:add", count=1, method_name="origin_visit_status" | "origin_visit_status:add", count=1, method_name="origin_visit_status" | ||||
| ) | ) | ||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_visit_status_add( | def origin_visit_status_add( | ||||
| self, visit_statuses: List[OriginVisitStatus], db=None, cur=None, | self, visit_statuses: List[OriginVisitStatus], db=None, cur=None, | ||||
| ) -> None: | ) -> None: | ||||
| visit_statuses_ = [] | |||||
| # First round to check existence (fail early if any is ko) | # First round to check existence (fail early if any is ko) | ||||
| for visit_status in visit_statuses: | for visit_status in visit_statuses: | ||||
| origin_url = self.origin_get([visit_status.origin], db=db, cur=cur)[0] | origin_url = self.origin_get([visit_status.origin], db=db, cur=cur)[0] | ||||
| if not origin_url: | if not origin_url: | ||||
| raise StorageArgumentException(f"Unknown origin {visit_status.origin}") | raise StorageArgumentException(f"Unknown origin {visit_status.origin}") | ||||
| for visit_status in visit_statuses: | if visit_status.type is None: | ||||
| origin_visit = self.origin_visit_get_by( | |||||
| visit_status.origin, visit_status.visit, db=db, cur=cur | |||||
| ) | |||||
| assert origin_visit is not None | |||||
| origin_visit_status = attr.evolve(visit_status, type=origin_visit.type) | |||||
| else: | |||||
| origin_visit_status = visit_status | |||||
| visit_statuses_.append(origin_visit_status) | |||||
| for visit_status in visit_statuses_: | |||||
| self._origin_visit_status_add(visit_status, db, cur) | self._origin_visit_status_add(visit_status, db, cur) | ||||
| @timed | @timed | ||||
| @db_transaction() | @db_transaction() | ||||
| def origin_visit_status_get_latest( | def origin_visit_status_get_latest( | ||||
| self, | self, | ||||
| origin_url: str, | origin_url: str, | ||||
| visit: int, | visit: int, | ||||
| ▲ Show 20 Lines • Show All 146 Lines • ▼ Show 20 Lines | ) -> PagedResult[OriginVisitStatus]: | ||||
| origin, visit, date_from=date_from, order=order, limit=limit + 1, cur=cur, | origin, visit, date_from=date_from, order=order, limit=limit + 1, cur=cur, | ||||
| ): | ): | ||||
| row_d = dict(zip(db.origin_visit_status_cols, row)) | row_d = dict(zip(db.origin_visit_status_cols, row)) | ||||
| visit_statuses.append( | visit_statuses.append( | ||||
| OriginVisitStatus( | OriginVisitStatus( | ||||
| origin=row_d["origin"], | origin=row_d["origin"], | ||||
| visit=row_d["visit"], | visit=row_d["visit"], | ||||
| date=row_d["date"], | date=row_d["date"], | ||||
| type=row_d["type"], | |||||
| status=row_d["status"], | status=row_d["status"], | ||||
| snapshot=row_d["snapshot"], | snapshot=row_d["snapshot"], | ||||
| metadata=row_d["metadata"], | metadata=row_d["metadata"], | ||||
| ) | ) | ||||
| ) | ) | ||||
| if len(visit_statuses) > limit: | if len(visit_statuses) > limit: | ||||
| # last visit status date is the next page token | # last visit status date is the next page token | ||||
| ▲ Show 20 Lines • Show All 374 Lines • Show Last 20 Lines | |||||