Changeset View
Changeset View
Standalone View
Standalone View
swh/storage/proxies/retry.py
| # Copyright (C) 2019-2021 The Software Heritage developers | # Copyright (C) 2019-2021 The Software Heritage developers | ||||
| # See the AUTHORS file at the top-level directory of this distribution | # See the AUTHORS file at the top-level directory of this distribution | ||||
| # License: GNU General Public License version 3, or any later version | # License: GNU General Public License version 3, or any later version | ||||
| # See top-level LICENSE file for more information | # See top-level LICENSE file for more information | ||||
| import logging | import logging | ||||
| import traceback | import traceback | ||||
| from tenacity import retry, stop_after_attempt, wait_random_exponential | from tenacity import RetryCallState, retry, stop_after_attempt, wait_random_exponential | ||||
| from tenacity.wait import wait_base | |||||
| from swh.core.api import TransientRemoteException | |||||
| from swh.storage import get_storage | from swh.storage import get_storage | ||||
| from swh.storage.exc import StorageArgumentException | from swh.storage.exc import StorageArgumentException | ||||
| from swh.storage.interface import StorageInterface | from swh.storage.interface import StorageInterface | ||||
| logger = logging.getLogger(__name__) | logger = logging.getLogger(__name__) | ||||
| def should_retry_adding(retry_state) -> bool: | def should_retry_adding(retry_state: RetryCallState) -> bool: | ||||
| """Retry if the error/exception is (probably) not about a caller error""" | """Retry if the error/exception is (probably) not about a caller error""" | ||||
| attempt = retry_state.outcome | attempt = retry_state.outcome | ||||
| assert attempt | |||||
| if attempt.failed: | if attempt.failed: | ||||
| error = attempt.exception() | error = attempt.exception() | ||||
| if isinstance(error, StorageArgumentException): | if isinstance(error, StorageArgumentException): | ||||
| # Exception is due to an invalid argument | # Exception is due to an invalid argument | ||||
| return False | return False | ||||
| elif isinstance(error, KeyboardInterrupt): | elif isinstance(error, KeyboardInterrupt): | ||||
| return False | return False | ||||
| Show All 14 Lines | if attempt.failed: | ||||
| }, | }, | ||||
| ) | ) | ||||
| return True | return True | ||||
| else: | else: | ||||
| # No exception | # No exception | ||||
| return False | return False | ||||
| class wait_transient_exceptions(wait_base): | |||||
| """Wait longer when servers return HTTP 503.""" | |||||
| def __init__(self, wait: float) -> None: | |||||
| self.wait = wait | |||||
| def __call__(self, retry_state: RetryCallState) -> float: | |||||
| attempt = retry_state.outcome | |||||
| assert attempt | |||||
| if attempt.failed and isinstance(attempt.exception(), TransientRemoteException): | |||||
| return self.wait | |||||
| else: | |||||
| return 0.0 | |||||
| swh_retry = retry( | swh_retry = retry( | ||||
| retry=should_retry_adding, | retry=should_retry_adding, | ||||
| wait=wait_random_exponential(multiplier=1, max=10), | wait=wait_random_exponential(multiplier=1, max=10) + wait_transient_exceptions(10), | ||||
| stop=stop_after_attempt(3), | stop=stop_after_attempt(3), | ||||
| reraise=True, | reraise=True, | ||||
| ) | ) | ||||
| def retry_function(storage, attribute_name): | def retry_function(storage, attribute_name): | ||||
| @swh_retry | @swh_retry | ||||
| def newf(*args, **kwargs): | def newf(*args, **kwargs): | ||||
| Show All 21 Lines | |||||