Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/cassandra/storage.py
| Show First 20 Lines • Show All 1,099 Lines • ▼ Show 20 Lines | ) -> Optional[OriginVisitStatus]: | ||||
| if allowed_statuses: | if allowed_statuses: | ||||
| rows = [row for row in rows if row.status in allowed_statuses] | rows = [row for row in rows if row.status in allowed_statuses] | ||||
| if require_snapshot: | if require_snapshot: | ||||
| rows = [row for row in rows if row.snapshot is not None] | rows = [row for row in rows if row.snapshot is not None] | ||||
| if not rows: | if not rows: | ||||
| return None | return None | ||||
| return converters.row_to_visit_status(rows[0]) | return converters.row_to_visit_status(rows[0]) | ||||
| def origin_visit_status_get_random( | def origin_visit_status_get_random(self, type: str) -> Optional[OriginVisitStatus]: | ||||
| self, type: str | |||||
| ) -> Optional[Tuple[OriginVisit, OriginVisitStatus]]: | |||||
| back_in_the_day = now() - datetime.timedelta(weeks=12) # 3 months back | back_in_the_day = now() - datetime.timedelta(weeks=12) # 3 months back | ||||
| # Random position to start iteration at | # Random position to start iteration at | ||||
| start_token = random.randint(TOKEN_BEGIN, TOKEN_END) | start_token = random.randint(TOKEN_BEGIN, TOKEN_END) | ||||
| # Iterator over all visits, ordered by token(origins) then visit_id | # Iterator over all visits, ordered by token(origins) then visit_id | ||||
| rows = self._cql_runner.origin_visit_iter(start_token) | rows = self._cql_runner.origin_visit_iter(start_token) | ||||
| for row in rows: | for row in rows: | ||||
| visit = converters.row_to_visit(row) | visit = converters.row_to_visit(row) | ||||
| visit_status = self._origin_visit_get_latest_status(visit) | visit_status = self._origin_visit_get_latest_status(visit) | ||||
| if visit.date > back_in_the_day and visit_status.status == "full": | if visit.date > back_in_the_day and visit_status.status == "full": | ||||
| return visit, visit_status | return visit_status | ||||
| return None | return None | ||||
| def stat_counters(self): | def stat_counters(self): | ||||
| rows = self._cql_runner.stat_counters() | rows = self._cql_runner.stat_counters() | ||||
| keys = ( | keys = ( | ||||
| "content", | "content", | ||||
| "directory", | "directory", | ||||
| "origin", | "origin", | ||||
| ▲ Show 20 Lines • Show All 206 Lines • Show Last 20 Lines | |||||