Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/algos/origin.py
# Copyright (C) 2019-2020 The Software Heritage developers | # Copyright (C) 2019-2020 The Software Heritage developers | ||||
# See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Iterator, List, Optional, Tuple | from typing import Iterator, List, Optional, Tuple | ||||
from swh.model.model import Origin, OriginVisit, OriginVisitStatus | from swh.model.model import Origin, OriginVisit, OriginVisitStatus | ||||
from swh.storage.interface import StorageInterface | from swh.storage.interface import ListOrder, StorageInterface | ||||
def iter_origins( | def iter_origins( | ||||
storage: StorageInterface, | storage: StorageInterface, | ||||
origin_from: int = 1, | origin_from: int = 1, | ||||
origin_to: Optional[int] = None, | origin_to: Optional[int] = None, | ||||
batch_size: int = 10000, | batch_size: int = 10000, | ||||
) -> Iterator[Origin]: | ) -> Iterator[Origin]: | ||||
▲ Show 20 Lines • Show All 72 Lines • ▼ Show 20 Lines | if visit: | ||||
origin_url, | origin_url, | ||||
visit.visit, | visit.visit, | ||||
allowed_statuses=allowed_statuses, | allowed_statuses=allowed_statuses, | ||||
require_snapshot=require_snapshot, | require_snapshot=require_snapshot, | ||||
) | ) | ||||
if visit_status: | if visit_status: | ||||
result = visit, visit_status | result = visit, visit_status | ||||
return result | return result | ||||
def iter_origin_visits( | |||||
storage: StorageInterface, origin: str, order: ListOrder = ListOrder.ASC | |||||
) -> Iterator[OriginVisit]: | |||||
"""Iter over origin visits from an origin | |||||
""" | |||||
next_page_token = None | |||||
while True: | |||||
page = storage.origin_visit_get(origin, order=order, page_token=next_page_token) | |||||
next_page_token = page.next_page_token | |||||
yield from page.results | |||||
if page.next_page_token is None: | |||||
break | |||||
def iter_origin_visit_statuses( | |||||
storage: StorageInterface, origin: str, visit: int, order: ListOrder = ListOrder.ASC | |||||
) -> Iterator[OriginVisitStatus]: | |||||
"""Iter over origin visit status from an origin visit | |||||
""" | |||||
next_page_token = None | |||||
while True: | |||||
page = storage.origin_visit_status_get( | |||||
origin, visit, order=order, page_token=next_page_token | |||||
) | |||||
next_page_token = page.next_page_token | |||||
yield from page.results | |||||
if next_page_token is None: | |||||
break |