Changeset View
Changeset View
Standalone View
Standalone View
swh/scheduler/interface.py
Show First 20 Lines • Show All 392 Lines • ▼ Show 20 Lines | def grab_next_visits( | ||||
self, | self, | ||||
visit_type: str, | visit_type: str, | ||||
count: int, | count: int, | ||||
policy: str, | policy: str, | ||||
timestamp: Optional[datetime.datetime] = None, | timestamp: Optional[datetime.datetime] = None, | ||||
scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7), | scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7), | ||||
failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14), | failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14), | ||||
not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31), | not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31), | ||||
tablesample: Optional[float] = None, | |||||
) -> List[ListedOrigin]: | ) -> List[ListedOrigin]: | ||||
"""Get at most the `count` next origins that need to be visited with | """Get at most the `count` next origins that need to be visited with | ||||
the `visit_type` loader according to the given scheduling `policy`. | the `visit_type` loader according to the given scheduling `policy`. | ||||
This will mark the origins as scheduled in the origin_visit_stats | This will mark the origins as scheduled in the origin_visit_stats | ||||
table, to avoid scheduling multiple visits to the same origin. | table, to avoid scheduling multiple visits to the same origin. | ||||
Arguments: | Arguments: | ||||
visit_type: type of visits to schedule | visit_type: type of visits to schedule | ||||
count: number of visits to schedule | count: number of visits to schedule | ||||
policy: the scheduling policy used to select which visits to schedule | policy: the scheduling policy used to select which visits to schedule | ||||
timestamp: the mocked timestamp at which we're recording that the visits are | timestamp: the mocked timestamp at which we're recording that the visits are | ||||
being scheduled (defaults to the current time) | being scheduled (defaults to the current time) | ||||
scheduled_cooldown: the minimal interval before which we can schedule | scheduled_cooldown: the minimal interval before which we can schedule | ||||
the same origin again | the same origin again | ||||
failed_cooldown: the minimal interval before which we can reschedule a | failed_cooldown: the minimal interval before which we can reschedule a | ||||
failed origin | failed origin | ||||
not_found_cooldown: the minimal interval before which we can reschedule a | not_found_cooldown: the minimal interval before which we can reschedule a | ||||
not_found origin | not_found origin | ||||
tablesample: the percentage of the table on which we run the query | |||||
(None: no sampling) | |||||
""" | """ | ||||
... | ... | ||||
@remote_api_endpoint("visit_stats/upsert") | @remote_api_endpoint("visit_stats/upsert") | ||||
def origin_visit_stats_upsert( | def origin_visit_stats_upsert( | ||||
self, origin_visit_stats: Iterable[OriginVisitStats] | self, origin_visit_stats: Iterable[OriginVisitStats] | ||||
) -> None: | ) -> None: | ||||
"""Create a new origin visit stats | """Create a new origin visit stats | ||||
▲ Show 20 Lines • Show All 62 Lines • Show Last 20 Lines |