Differential D5818 Diff 22038 swh/scheduler/interface.py

Changeset View

Standalone View

swh/scheduler/interface.py

Show First 20 Lines • Show All 388 Lines • ▼ Show 20 Lines	) -> PaginatedListedOriginList:
...		...

@remote_api_endpoint("origins/grab_next")		@remote_api_endpoint("origins/grab_next")
def grab_next_visits(		def grab_next_visits(
self,		self,
visit_type: str,		visit_type: str,
count: int,		count: int,
policy: str,		policy: str,
		enabled: bool = True,
		lister_uuid: Optional[str] = None,
timestamp: Optional[datetime.datetime] = None,		timestamp: Optional[datetime.datetime] = None,
scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7),		scheduled_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=7),
failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14),		failed_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=14),
not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31),		not_found_cooldown: Optional[datetime.timedelta] = datetime.timedelta(days=31),
tablesample: Optional[float] = None,		tablesample: Optional[float] = None,
) -> List[ListedOrigin]:		) -> List[ListedOrigin]:
"""Get at most the `count` next origins that need to be visited with		"""Get at most the `count` next origins that need to be visited with
the `visit_type` loader according to the given scheduling `policy`.		the `visit_type` loader according to the given scheduling `policy`.

This will mark the origins as scheduled in the origin_visit_stats		This will mark the origins as scheduled in the origin_visit_stats
table, to avoid scheduling multiple visits to the same origin.		table, to avoid scheduling multiple visits to the same origin.

Arguments:		Arguments:
visit_type: type of visits to schedule		visit_type: type of visits to schedule
count: number of visits to schedule		count: number of visits to schedule
policy: the scheduling policy used to select which visits to schedule		policy: the scheduling policy used to select which visits to schedule
		enabled: Determine whether we want to list enabled or disabled origins. As
		default, we want reasonably enabled origins. For some edge case, we might
		want the others.
		lister_uuid: Determine the list of origins listed from the lister with uuid
timestamp: the mocked timestamp at which we're recording that the visits are		timestamp: the mocked timestamp at which we're recording that the visits are
being scheduled (defaults to the current time)		being scheduled (defaults to the current time)
scheduled_cooldown: the minimal interval before which we can schedule		scheduled_cooldown: the minimal interval before which we can schedule
the same origin again		the same origin again
failed_cooldown: the minimal interval before which we can reschedule a		failed_cooldown: the minimal interval before which we can reschedule a
failed origin		failed origin
not_found_cooldown: the minimal interval before which we can reschedule a		not_found_cooldown: the minimal interval before which we can reschedule a
not_found origin		not_found origin
tablesample: the percentage of the table on which we run the query		tablesample: the percentage of the table on which we run the query
(None: no sampling)		(None: no sampling)

"""		"""
...		...

@remote_api_endpoint("visit_stats/upsert")		@remote_api_endpoint("visit_stats/upsert")
def origin_visit_stats_upsert(		def origin_visit_stats_upsert(
self, origin_visit_stats: Iterable[OriginVisitStats]		self, origin_visit_stats: Iterable[OriginVisitStats]
) -> None:		) -> None:
"""Create a new origin visit stats		"""Create a new origin visit stats
▲ Show 20 Lines • Show All 62 Lines • Show Last 20 Lines