Changeset View
Changeset View
Standalone View
Standalone View
swh/search/utils.py
# Copyright (C) 2019 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
def stream_results(f, *args, **kwargs): | def stream_results(f, *args, **kwargs): | ||||
"""Consume the paginated result and stream it directly | |||||
""" | |||||
if "page_token" in kwargs: | if "page_token" in kwargs: | ||||
raise TypeError('stream_results has no argument "page_token".') | raise TypeError('stream_results has no argument "page_token".') | ||||
page_token = None | page_token = None | ||||
while True: | while True: | ||||
results = f(*args, page_token=page_token, **kwargs) | results = f(*args, page_token=page_token, **kwargs) | ||||
yield from results["results"] | yield from results.results | ||||
page_token = results["next_page_token"] | page_token = results.next_page_token | ||||
if page_token is None: | if page_token is None: | ||||
break | break | ||||
ardumont: Maybe that can go one module down in swh.core now.
It's shared behavior with storage now (swh. | |||||
Not Done Inline ActionsYeah, I have this on my mental todo-list vlorentz: Yeah, I have this on my mental todo-list | |||||
Done Inline Actionsardumont: D3664 |
Maybe that can go one module down in swh.core now.
It's shared behavior with storage now (swh.storage.algos.origin.iter_origin_visit and iter_origin_visit_status comes to mind...)