Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/algos/origin.py
| # Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| from typing import Iterator, List, Optional, Tuple | from typing import Iterator, List, Optional, Tuple | ||||
| from swh.model.model import Origin, OriginVisit, OriginVisitStatus | from swh.model.model import Origin, OriginVisit, OriginVisitStatus | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import ListOrder, StorageInterface | ||||
| def iter_origins( | def iter_origins( | ||||
| storage: StorageInterface, | storage: StorageInterface, | ||||
| origin_from: int = 1, | origin_from: int = 1, | ||||
| origin_to: Optional[int] = None, | origin_to: Optional[int] = None, | ||||
| batch_size: int = 10000, | batch_size: int = 10000, | ||||
| ) -> Iterator[Origin]: | ) -> Iterator[Origin]: | ||||
| ▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | if visit: | ||||
| origin_url, | origin_url, | ||||
| visit.visit, | visit.visit, | ||||
| allowed_statuses=allowed_statuses, | allowed_statuses=allowed_statuses, | ||||
| require_snapshot=require_snapshot, | require_snapshot=require_snapshot, | ||||
| ) | ) | ||||
| if visit_status: | if visit_status: | ||||
| result = visit, visit_status | result = visit, visit_status | ||||
| return result | return result | ||||
| def iter_origin_visits( | |||||
| storage: StorageInterface, origin: str, order: ListOrder = ListOrder.ASC | |||||
| ) -> Iterator[OriginVisit]: | |||||
| """Iter over origin visits from an origin | |||||
| """ | |||||
| next_page_token = None | |||||
| while True: | |||||
| page = storage.origin_visit_get(origin, order=order, page_token=next_page_token) | |||||
| next_page_token = page.next_page_token | |||||
| yield from page.results | |||||
| if page.next_page_token is None: | |||||
| break | |||||
| def iter_origin_visit_statuses( | |||||
| storage: StorageInterface, origin: str, visit: int, order: ListOrder = ListOrder.ASC | |||||
| ) -> Iterator[OriginVisitStatus]: | |||||
| """Iter over origin visit status from an origin visit | |||||
| """ | |||||
| next_page_token = None | |||||
| while True: | |||||
| page = storage.origin_visit_status_get( | |||||
| origin, visit, order=order, page_token=next_page_token | |||||
| ) | |||||
| next_page_token = page.next_page_token | |||||
| yield from page.results | |||||
| if next_page_token is None: | |||||
| break | |||||