Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/backend.py
| Show First 20 Lines • Show All 371 Lines • ▼ Show 20 Lines | class SchedulerBackend: | ||||
| def grab_next_visits( | def grab_next_visits( | ||||
| self, | self, | ||||
| visit_type: str, | visit_type: str, | ||||
| count: int, | count: int, | ||||
| policy: str, | policy: str, | ||||
| enabled: bool = True, | enabled: bool = True, | ||||
| lister_uuid: Optional[str] = None, | lister_uuid: Optional[str] = None, | ||||
| timestamp: Optional[datetime.datetime] = None, | timestamp: Optional[datetime.datetime] = None, | ||||
| absolute_cooldown: Optional[datetime.timedelta] = datetime.timedelta(hours=12), | |||||
| scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7), | scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7), | ||||
| failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14), | failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14), | ||||
| not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31), | not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31), | ||||
| tablesample: Optional[float] = None, | tablesample: Optional[float] = None, | ||||
| db=None, | db=None, | ||||
| cur=None, | cur=None, | ||||
| ) -> List[ListedOrigin]: | ) -> List[ListedOrigin]: | ||||
| if timestamp is None: | if timestamp is None: | ||||
| Show All 10 Lines | ) -> List[ListedOrigin]: | ||||
| # "NOT enabled" = the lister said the origin no longer exists | # "NOT enabled" = the lister said the origin no longer exists | ||||
| where_clauses.append("enabled" if enabled else "not enabled") | where_clauses.append("enabled" if enabled else "not enabled") | ||||
| # Only schedule visits of the given type | # Only schedule visits of the given type | ||||
| where_clauses.append("visit_type = %s") | where_clauses.append("visit_type = %s") | ||||
| query_args.append(visit_type) | query_args.append(visit_type) | ||||
| if absolute_cooldown: | |||||
| # Don't schedule visits if they've been scheduled since the absolute cooldown | |||||
| where_clauses.append( | |||||
| """origin_visit_stats.last_scheduled IS NULL | |||||
| OR origin_visit_stats.last_scheduled < %s | |||||
| """ | |||||
| ) | |||||
| query_args.append(timestamp - absolute_cooldown) | |||||
| if scheduled_cooldown: | if scheduled_cooldown: | ||||
| # Don't re-schedule visits if they're already scheduled but we haven't | # Don't re-schedule visits if they're already scheduled but we haven't | ||||
| # recorded a result yet, unless they've been scheduled more than a week | # recorded a result yet, unless they've been scheduled more than a week | ||||
| # ago (it probably means we've lost them in flight somewhere). | # ago (it probably means we've lost them in flight somewhere). | ||||
| where_clauses.append( | where_clauses.append( | ||||
| """origin_visit_stats.last_scheduled IS NULL | """origin_visit_stats.last_scheduled IS NULL | ||||
| OR origin_visit_stats.last_scheduled < GREATEST( | OR origin_visit_stats.last_scheduled < GREATEST( | ||||
| %s, | %s, | ||||
| ▲ Show 20 Lines • Show All 733 Lines • Show Last 20 Lines | |||||