Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/pattern.py
Show First 20 Lines • Show All 217 Lines • ▼ Show 20 Lines | class Lister(Generic[StateType, PageType]): | ||||
def send_origins(self, origins: Iterable[model.ListedOrigin]) -> int: | def send_origins(self, origins: Iterable[model.ListedOrigin]) -> int: | ||||
"""Record a list of :class:`model.ListedOrigin` in the scheduler. | """Record a list of :class:`model.ListedOrigin` in the scheduler. | ||||
Returns: | Returns: | ||||
the number of listed origins recorded in the scheduler | the number of listed origins recorded in the scheduler | ||||
""" | """ | ||||
count = 0 | count = 0 | ||||
for batch_origins in grouper(origins, n=100): | for batch_origins in grouper(origins, n=1000): | ||||
ret = self.scheduler.record_listed_origins(batch_origins) | ret = self.scheduler.record_listed_origins(batch_origins) | ||||
count += len(ret) | count += len(ret) | ||||
return count | return count | ||||
@classmethod | @classmethod | ||||
def from_config(cls, scheduler: Dict[str, Any], **config: Any): | def from_config(cls, scheduler: Dict[str, Any], **config: Any): | ||||
"""Instantiate a lister from a configuration dict. | """Instantiate a lister from a configuration dict. | ||||
▲ Show 20 Lines • Show All 43 Lines • Show Last 20 Lines |