Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
| Show First 20 Lines • Show All 372 Lines • ▼ Show 20 Lines | ) -> List[ListedOrigin]: | ||||
| if scheduled_cooldown: | if scheduled_cooldown: | ||||
| # Don't re-schedule visits if they're already scheduled but we haven't | # Don't re-schedule visits if they're already scheduled but we haven't | ||||
| # recorded a result yet, unless they've been scheduled more than a week | # recorded a result yet, unless they've been scheduled more than a week | ||||
| # ago (it probably means we've lost them in flight somewhere). | # ago (it probably means we've lost them in flight somewhere). | ||||
| where_clauses.append( | where_clauses.append( | ||||
| """origin_visit_stats.last_scheduled IS NULL | """origin_visit_stats.last_scheduled IS NULL | ||||
| OR origin_visit_stats.last_scheduled < GREATEST( | OR origin_visit_stats.last_scheduled < GREATEST( | ||||
| %s - %s, | %s, | ||||
| origin_visit_stats.last_visit | origin_visit_stats.last_visit | ||||
| ) | ) | ||||
| """ | """ | ||||
| ) | ) | ||||
| query_args.append(timestamp) | query_args.append(timestamp - scheduled_cooldown) | ||||
| query_args.append(scheduled_cooldown) | |||||
| if failed_cooldown: | if failed_cooldown: | ||||
| # Don't retry failed origins too often | # Don't retry failed origins too often | ||||
| where_clauses.append( | where_clauses.append( | ||||
| "origin_visit_stats.last_visit_status is distinct from 'failed' " | "origin_visit_stats.last_visit_status is distinct from 'failed' " | ||||
| "or origin_visit_stats.last_visit < %s - %s" | "or origin_visit_stats.last_visit < %s" | ||||
| ) | ) | ||||
| query_args.append(timestamp) | query_args.append(timestamp - failed_cooldown) | ||||
| query_args.append(failed_cooldown) | |||||
| if not_found_cooldown: | if not_found_cooldown: | ||||
| # Don't retry not found origins too often | # Don't retry not found origins too often | ||||
| where_clauses.append( | where_clauses.append( | ||||
| "origin_visit_stats.last_visit_status is distinct from 'not_found' " | "origin_visit_stats.last_visit_status is distinct from 'not_found' " | ||||
| "or origin_visit_stats.last_visit < %s - %s" | "or origin_visit_stats.last_visit < %s" | ||||
| ) | ) | ||||
| query_args.append(timestamp) | query_args.append(timestamp - not_found_cooldown) | ||||
| query_args.append(not_found_cooldown) | |||||
| if policy == "oldest_scheduled_first": | if policy == "oldest_scheduled_first": | ||||
| order_by = "origin_visit_stats.last_scheduled NULLS FIRST" | order_by = "origin_visit_stats.last_scheduled NULLS FIRST" | ||||
| elif policy == "never_visited_oldest_update_first": | elif policy == "never_visited_oldest_update_first": | ||||
| # never visited origins have a NULL last_snapshot | # never visited origins have a NULL last_snapshot | ||||
| where_clauses.append("origin_visit_stats.last_snapshot IS NULL") | where_clauses.append("origin_visit_stats.last_snapshot IS NULL") | ||||
| # order by increasing last_update (oldest first) | # order by increasing last_update (oldest first) | ||||
| ▲ Show 20 Lines • Show All 692 Lines • Show Last 20 Lines | |||||