Spaces:
Build error
Build error
File size: 7,893 Bytes
64772a4 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 |
import abc
import random
from typing import Any, Awaitable, Callable, Iterable, List, Optional, Set, Type
from warnings import warn
from aiohttp import ClientResponse
EvaluateResponseCallbackType = Callable[[ClientResponse], Awaitable[bool]]
class RetryOptionsBase:
def __init__(
self,
attempts: int = 3, # How many times we should retry
statuses: Optional[Iterable[int]] = None, # On which statuses we should retry
exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry
retry_all_server_errors: bool = True, # If should retry all 500 errors or not
# a callback that will run on response to decide if retry
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
self.attempts: int = attempts
if statuses is None:
statuses = set()
self.statuses: Iterable[int] = statuses
if exceptions is None:
exceptions = set()
self.exceptions: Iterable[Type[Exception]] = exceptions
self.retry_all_server_errors = retry_all_server_errors
self.evaluate_response_callback = evaluate_response_callback
@abc.abstractmethod
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
raise NotImplementedError
class ExponentialRetry(RetryOptionsBase):
def __init__(
self,
attempts: int = 3, # How many times we should retry
start_timeout: float = 0.1, # Base timeout time, then it exponentially grow
max_timeout: float = 30.0, # Max possible timeout between tries
factor: float = 2.0, # How much we increase timeout each time
statuses: Optional[Set[int]] = None, # On which statuses we should retry
exceptions: Optional[Set[Type[Exception]]] = None, # On which exceptions we should retry
retry_all_server_errors: bool = True,
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
super().__init__(
attempts=attempts,
statuses=statuses,
exceptions=exceptions,
retry_all_server_errors=retry_all_server_errors,
evaluate_response_callback=evaluate_response_callback,
)
self._start_timeout: float = start_timeout
self._max_timeout: float = max_timeout
self._factor: float = factor
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
"""Return timeout with exponential backoff."""
timeout = self._start_timeout * (self._factor ** attempt)
return min(timeout, self._max_timeout)
def RetryOptions(*args: Any, **kwargs: Any) -> ExponentialRetry:
warn("RetryOptions is deprecated, use ExponentialRetry")
return ExponentialRetry(*args, **kwargs)
class RandomRetry(RetryOptionsBase):
def __init__(
self,
attempts: int = 3, # How many times we should retry
statuses: Optional[Iterable[int]] = None, # On which statuses we should retry
exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry
min_timeout: float = 0.1, # Minimum possible timeout
max_timeout: float = 3.0, # Maximum possible timeout between tries
random_func: Callable[[], float] = random.random, # Random number generator
retry_all_server_errors: bool = True,
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
super().__init__(
attempts=attempts,
statuses=statuses,
exceptions=exceptions,
retry_all_server_errors=retry_all_server_errors,
evaluate_response_callback=evaluate_response_callback,
)
self.attempts: int = attempts
self.min_timeout: float = min_timeout
self.max_timeout: float = max_timeout
self.random = random_func
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
"""Generate random timeouts."""
return self.min_timeout + self.random() * (self.max_timeout - self.min_timeout)
class ListRetry(RetryOptionsBase):
def __init__(
self,
timeouts: List[float],
statuses: Optional[Iterable[int]] = None, # On which statuses we should retry
exceptions: Optional[Iterable[Type[Exception]]] = None, # On which exceptions we should retry
retry_all_server_errors: bool = True,
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
super().__init__(
attempts=len(timeouts),
statuses=statuses,
exceptions=exceptions,
retry_all_server_errors=retry_all_server_errors,
evaluate_response_callback=evaluate_response_callback,
)
self.timeouts = timeouts
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
"""timeouts from a defined list."""
return self.timeouts[attempt]
class FibonacciRetry(RetryOptionsBase):
def __init__(
self,
attempts: int = 3,
multiplier: float = 1.0,
statuses: Optional[Iterable[int]] = None,
exceptions: Optional[Iterable[Type[Exception]]] = None,
max_timeout: float = 3.0, # Maximum possible timeout between tries
retry_all_server_errors: bool = True,
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
super().__init__(
attempts=attempts,
statuses=statuses,
exceptions=exceptions,
retry_all_server_errors=retry_all_server_errors,
evaluate_response_callback=evaluate_response_callback,
)
self.max_timeout = max_timeout
self.multiplier = multiplier
self.prev_step = 1.0
self.current_step = 1.0
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
new_current_step = self.prev_step + self.current_step
self.prev_step = self.current_step
self.current_step = new_current_step
return min(self.multiplier * new_current_step, self.max_timeout)
class JitterRetry(ExponentialRetry):
"""https://github.com/inyutin/aiohttp_retry/issues/44"""
def __init__(
self,
attempts: int = 3, # How many times we should retry
start_timeout: float = 0.1, # Base timeout time, then it exponentially grow
max_timeout: float = 30.0, # Max possible timeout between tries
factor: float = 2.0, # How much we increase timeout each time
statuses: Optional[Set[int]] = None, # On which statuses we should retry
exceptions: Optional[Set[Type[Exception]]] = None, # On which exceptions we should retry
random_interval_size: float = 2.0, # size of interval for random component
retry_all_server_errors: bool = True,
evaluate_response_callback: Optional[EvaluateResponseCallbackType] = None,
):
super().__init__(
attempts=attempts,
start_timeout=start_timeout,
max_timeout=max_timeout,
factor=factor,
statuses=statuses,
exceptions=exceptions,
retry_all_server_errors=retry_all_server_errors,
evaluate_response_callback=evaluate_response_callback,
)
self._start_timeout: float = start_timeout
self._max_timeout: float = max_timeout
self._factor: float = factor
self._random_interval_size = random_interval_size
def get_timeout(self, attempt: int, response: Optional[ClientResponse] = None) -> float:
timeout: float = super().get_timeout(attempt) + random.uniform(0, self._random_interval_size) ** self._factor
return timeout
|