Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
| Show First 20 Lines • Show All 336 Lines • ▼ Show 20 Lines | ) -> List[ListedOrigin]: | ||||
| order_by = "origin_visit_stats.last_scheduled NULLS FIRST" | order_by = "origin_visit_stats.last_scheduled NULLS FIRST" | ||||
| elif policy == "never_visited_oldest_update_first": | elif policy == "never_visited_oldest_update_first": | ||||
| # never visited origins have a NULL last_snapshot | # never visited origins have a NULL last_snapshot | ||||
| where_clauses.append("origin_visit_stats.last_snapshot IS NULL") | where_clauses.append("origin_visit_stats.last_snapshot IS NULL") | ||||
| # order by increasing last_update (oldest first) | # order by increasing last_update (oldest first) | ||||
| where_clauses.append("listed_origins.last_update IS NOT NULL") | where_clauses.append("listed_origins.last_update IS NOT NULL") | ||||
| order_by = "listed_origins.last_update" | order_by = "listed_origins.last_update" | ||||
| elif policy == "already_visited_order_by_lag": | |||||
| # TODO: store "visit lag" in a materialized view? | |||||
| # visited origins have a NOT NULL last_snapshot | |||||
| where_clauses.append("origin_visit_stats.last_snapshot IS NOT NULL") | |||||
| # ignore origins we have visited after the known last update | |||||
| where_clauses.append("listed_origins.last_update IS NOT NULL") | |||||
| where_clauses.append( | |||||
| """ | |||||
| listed_origins.last_update | |||||
| > GREATEST( | |||||
| origin_visit_stats.last_eventful, | |||||
| origin_visit_stats.last_uneventful | |||||
| ) | |||||
| """ | |||||
| ) | |||||
| # order by decreasing visit lag | |||||
| order_by = """\ | |||||
| listed_origins.last_update | |||||
| - GREATEST( | |||||
| origin_visit_stats.last_eventful, | |||||
| origin_visit_stats.last_uneventful | |||||
| ) | |||||
| DESC | |||||
| """ | |||||
| else: | else: | ||||
| raise UnknownPolicy(f"Unknown scheduling policy {policy}") | raise UnknownPolicy(f"Unknown scheduling policy {policy}") | ||||
| select_query = f""" | select_query = f""" | ||||
| SELECT | SELECT | ||||
| {origin_select_cols} | {origin_select_cols} | ||||
| FROM | FROM | ||||
| listed_origins | listed_origins | ||||
| ▲ Show 20 Lines • Show All 611 Lines • Show Last 20 Lines | |||||