Changeset View
Changeset View
Standalone View
Standalone View
swh/lister/utils.py
# Copyright (C) 2018-2021 the Software Heritage developers | # Copyright (C) 2018-2022 the Software Heritage developers | ||||
# License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
# See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
from typing import Callable, Iterator, Tuple | from typing import Callable, Iterator, Tuple | ||||
from requests.exceptions import ConnectionError, HTTPError | from requests.exceptions import ConnectionError, HTTPError | ||||
from requests.status_codes import codes | from requests.status_codes import codes | ||||
from tenacity import retry as tenacity_retry | from tenacity import retry as tenacity_retry | ||||
▲ Show 20 Lines • Show All 51 Lines • ▼ Show 20 Lines | def retry_if_exception(retry_state, predicate: Callable[[Exception], bool]) -> bool: | ||||
""" | """ | ||||
attempt = retry_state.outcome | attempt = retry_state.outcome | ||||
if attempt.failed: | if attempt.failed: | ||||
exception = attempt.exception() | exception = attempt.exception() | ||||
return predicate(exception) | return predicate(exception) | ||||
return False | return False | ||||
def retry_if_throttling(retry_state) -> bool: | |||||
""" | |||||
Custom tenacity retry predicate for handling HTTP responses with | |||||
status code 429 (too many requests). | |||||
""" | |||||
return retry_if_exception(retry_state, is_throttling_exception) | |||||
def retry_policy_generic(retry_state) -> bool: | def retry_policy_generic(retry_state) -> bool: | ||||
""" | """ | ||||
Custom tenacity retry predicate for handling failed requests: | Custom tenacity retry predicate for handling failed requests: | ||||
- ConnectionError | - ConnectionError | ||||
- Server errors (status >= 500) | - Server errors (status >= 500) | ||||
- Throttling errors (status == 429) | - Throttling errors (status == 429) | ||||
This does not handle 404, 403 or other status codes. | This does not handle 404, 403 or other status codes. | ||||
""" | """ | ||||
return retry_if_exception(retry_state, is_retryable_exception) | return retry_if_exception(retry_state, is_retryable_exception) | ||||
WAIT_EXP_BASE = 10 | WAIT_EXP_BASE = 10 | ||||
MAX_NUMBER_ATTEMPTS = 5 | MAX_NUMBER_ATTEMPTS = 5 | ||||
def throttling_retry( | def http_retry( | ||||
retry=retry_if_throttling, | retry=retry_policy_generic, | ||||
wait=wait_exponential(exp_base=WAIT_EXP_BASE), | wait=wait_exponential(exp_base=WAIT_EXP_BASE), | ||||
stop=stop_after_attempt(max_attempt_number=MAX_NUMBER_ATTEMPTS), | stop=stop_after_attempt(max_attempt_number=MAX_NUMBER_ATTEMPTS), | ||||
**retry_args, | **retry_args, | ||||
): | ): | ||||
""" | """ | ||||
Decorator based on `tenacity` for retrying a function possibly raising | Decorator based on `tenacity` for retrying a function possibly raising | ||||
requests.exception.HTTPError for status code 429 (too many requests). | requests.exception.HTTPError for status code 429 (too many requests). | ||||
Show All 19 Lines |