Differential D4899 Diff 17517 swh/scheduler/backend.py

Changeset View

Standalone View

swh/scheduler/backend.py

Show First 20 Lines • Show All 336 Lines • ▼ Show 20 Lines	) -> List[ListedOrigin]:
order_by = "origin_visit_stats.last_scheduled NULLS FIRST"		order_by = "origin_visit_stats.last_scheduled NULLS FIRST"
elif policy == "never_visited_oldest_update_first":		elif policy == "never_visited_oldest_update_first":
# never visited origins have a NULL last_snapshot		# never visited origins have a NULL last_snapshot
where_clauses.append("origin_visit_stats.last_snapshot IS NULL")		where_clauses.append("origin_visit_stats.last_snapshot IS NULL")

# order by increasing last_update (oldest first)		# order by increasing last_update (oldest first)
where_clauses.append("listed_origins.last_update IS NOT NULL")		where_clauses.append("listed_origins.last_update IS NOT NULL")
order_by = "listed_origins.last_update"		order_by = "listed_origins.last_update"
		elif policy == "already_visited_order_by_lag":
		# TODO: store "visit lag" in a materialized view?

		# visited origins have a NOT NULL last_snapshot
		where_clauses.append("origin_visit_stats.last_snapshot IS NOT NULL")

		# ignore origins we have visited after the known last update
		where_clauses.append("listed_origins.last_update IS NOT NULL")
		where_clauses.append(
		"""
		listed_origins.last_update
		> GREATEST(
		origin_visit_stats.last_eventful,
		origin_visit_stats.last_uneventful
		)
		"""
		)

		# order by decreasing visit lag
		order_by = """\
		listed_origins.last_update
		- GREATEST(
		origin_visit_stats.last_eventful,
		origin_visit_stats.last_uneventful
		)
		DESC
		"""
else:		else:
raise UnknownPolicy(f"Unknown scheduling policy {policy}")		raise UnknownPolicy(f"Unknown scheduling policy {policy}")

select_query = f"""		select_query = f"""
SELECT		SELECT
{origin_select_cols}		{origin_select_cols}
FROM		FROM
listed_origins		listed_origins
▲ Show 20 Lines • Show All 611 Lines • Show Last 20 Lines