Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/gitlab/lister.py
| Show All 11 Lines | |||||
| import iso8601 | import iso8601 | ||||
| import requests | import requests | ||||
| from requests.exceptions import HTTPError | from requests.exceptions import HTTPError | ||||
| from requests.status_codes import codes | from requests.status_codes import codes | ||||
| from tenacity.before_sleep import before_sleep_log | from tenacity.before_sleep import before_sleep_log | ||||
| from swh.lister import USER_AGENT | from swh.lister import USER_AGENT | ||||
| from swh.lister.pattern import CredentialsType, Lister | from swh.lister.pattern import CredentialsType, Lister | ||||
| from swh.lister.utils import retry_attempt, throttling_retry | from swh.lister.utils import is_retryable_exception, retry_attempt, throttling_retry | ||||
| from swh.scheduler.model import ListedOrigin | from swh.scheduler.model import ListedOrigin | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| @dataclass | @dataclass | ||||
| class GitLabListerState: | class GitLabListerState: | ||||
| """State of the GitLabLister""" | """State of the GitLabLister""" | ||||
| Show All 22 Lines | def _if_rate_limited(retry_state) -> bool: | ||||
| """ | """ | ||||
| attempt = retry_attempt(retry_state) | attempt = retry_attempt(retry_state) | ||||
| if attempt.failed: | if attempt.failed: | ||||
| exc = attempt.exception() | exc = attempt.exception() | ||||
| return ( | return ( | ||||
| isinstance(exc, HTTPError) | isinstance(exc, HTTPError) | ||||
| and exc.response.status_code == codes.forbidden | and exc.response.status_code == codes.forbidden | ||||
| and int(exc.response.headers.get("RateLimit-Remaining", "0")) == 0 | and int(exc.response.headers.get("RateLimit-Remaining", "0")) == 0 | ||||
| ) | ) or is_retryable_exception(exc) | ||||
| return False | return False | ||||
| def _parse_id_after(url: Optional[str]) -> Optional[int]: | def _parse_id_after(url: Optional[str]) -> Optional[int]: | ||||
| """Given an url, extract a return the 'id_after' query parameter associated value | """Given an url, extract a return the 'id_after' query parameter associated value | ||||
| or None. | or None. | ||||
| This is the the repository id used for pagination purposes. | This is the the repository id used for pagination purposes. | ||||
| ▲ Show 20 Lines • Show All 170 Lines • Show Last 20 Lines | |||||