You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

267 lines
9.7 KiB

4 years ago
  1. ## Copyright 2013-2014 Ray Holder
  2. ##
  3. ## Licensed under the Apache License, Version 2.0 (the "License");
  4. ## you may not use this file except in compliance with the License.
  5. ## You may obtain a copy of the License at
  6. ##
  7. ## http://www.apache.org/licenses/LICENSE-2.0
  8. ##
  9. ## Unless required by applicable law or agreed to in writing, software
  10. ## distributed under the License is distributed on an "AS IS" BASIS,
  11. ## WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. ## See the License for the specific language governing permissions and
  13. ## limitations under the License.
  14. import random
  15. import six
  16. import sys
  17. import time
  18. import traceback
  19. # sys.maxint / 2, since Python 3.2 doesn't have a sys.maxint...
  20. MAX_WAIT = 1073741823
  21. def retry(*dargs, **dkw):
  22. """
  23. Decorator function that instantiates the Retrying object
  24. @param *dargs: positional arguments passed to Retrying object
  25. @param **dkw: keyword arguments passed to the Retrying object
  26. """
  27. # support both @retry and @retry() as valid syntax
  28. if len(dargs) == 1 and callable(dargs[0]):
  29. def wrap_simple(f):
  30. @six.wraps(f)
  31. def wrapped_f(*args, **kw):
  32. return Retrying().call(f, *args, **kw)
  33. return wrapped_f
  34. return wrap_simple(dargs[0])
  35. else:
  36. def wrap(f):
  37. @six.wraps(f)
  38. def wrapped_f(*args, **kw):
  39. return Retrying(*dargs, **dkw).call(f, *args, **kw)
  40. return wrapped_f
  41. return wrap
  42. class Retrying(object):
  43. def __init__(self,
  44. stop=None, wait=None,
  45. stop_max_attempt_number=None,
  46. stop_max_delay=None,
  47. wait_fixed=None,
  48. wait_random_min=None, wait_random_max=None,
  49. wait_incrementing_start=None, wait_incrementing_increment=None,
  50. wait_exponential_multiplier=None, wait_exponential_max=None,
  51. retry_on_exception=None,
  52. retry_on_result=None,
  53. wrap_exception=False,
  54. stop_func=None,
  55. wait_func=None,
  56. wait_jitter_max=None):
  57. self._stop_max_attempt_number = 5 if stop_max_attempt_number is None else stop_max_attempt_number
  58. self._stop_max_delay = 100 if stop_max_delay is None else stop_max_delay
  59. self._wait_fixed = 1000 if wait_fixed is None else wait_fixed
  60. self._wait_random_min = 0 if wait_random_min is None else wait_random_min
  61. self._wait_random_max = 1000 if wait_random_max is None else wait_random_max
  62. self._wait_incrementing_start = 0 if wait_incrementing_start is None else wait_incrementing_start
  63. self._wait_incrementing_increment = 100 if wait_incrementing_increment is None else wait_incrementing_increment
  64. self._wait_exponential_multiplier = 1 if wait_exponential_multiplier is None else wait_exponential_multiplier
  65. self._wait_exponential_max = MAX_WAIT if wait_exponential_max is None else wait_exponential_max
  66. self._wait_jitter_max = 0 if wait_jitter_max is None else wait_jitter_max
  67. # TODO add chaining of stop behaviors
  68. # stop behavior
  69. stop_funcs = []
  70. if stop_max_attempt_number is not None:
  71. stop_funcs.append(self.stop_after_attempt)
  72. if stop_max_delay is not None:
  73. stop_funcs.append(self.stop_after_delay)
  74. if stop_func is not None:
  75. self.stop = stop_func
  76. elif stop is None:
  77. self.stop = lambda attempts, delay: any(f(attempts, delay) for f in stop_funcs)
  78. else:
  79. self.stop = getattr(self, stop)
  80. # TODO add chaining of wait behaviors
  81. # wait behavior
  82. wait_funcs = [lambda *args, **kwargs: 0]
  83. if wait_fixed is not None:
  84. wait_funcs.append(self.fixed_sleep)
  85. if wait_random_min is not None or wait_random_max is not None:
  86. wait_funcs.append(self.random_sleep)
  87. if wait_incrementing_start is not None or wait_incrementing_increment is not None:
  88. wait_funcs.append(self.incrementing_sleep)
  89. if wait_exponential_multiplier is not None or wait_exponential_max is not None:
  90. wait_funcs.append(self.exponential_sleep)
  91. if wait_func is not None:
  92. self.wait = wait_func
  93. elif wait is None:
  94. self.wait = lambda attempts, delay: max(f(attempts, delay) for f in wait_funcs)
  95. else:
  96. self.wait = getattr(self, wait)
  97. # retry on exception filter
  98. if retry_on_exception is None:
  99. self._retry_on_exception = self.always_reject
  100. else:
  101. self._retry_on_exception = retry_on_exception
  102. # TODO simplify retrying by Exception types
  103. # retry on result filter
  104. if retry_on_result is None:
  105. self._retry_on_result = self.never_reject
  106. else:
  107. self._retry_on_result = retry_on_result
  108. self._wrap_exception = wrap_exception
  109. def stop_after_attempt(self, previous_attempt_number, delay_since_first_attempt_ms):
  110. """Stop after the previous attempt >= stop_max_attempt_number."""
  111. return previous_attempt_number >= self._stop_max_attempt_number
  112. def stop_after_delay(self, previous_attempt_number, delay_since_first_attempt_ms):
  113. """Stop after the time from the first attempt >= stop_max_delay."""
  114. return delay_since_first_attempt_ms >= self._stop_max_delay
  115. def no_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
  116. """Don't sleep at all before retrying."""
  117. return 0
  118. def fixed_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
  119. """Sleep a fixed amount of time between each retry."""
  120. return self._wait_fixed
  121. def random_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
  122. """Sleep a random amount of time between wait_random_min and wait_random_max"""
  123. return random.randint(self._wait_random_min, self._wait_random_max)
  124. def incrementing_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
  125. """
  126. Sleep an incremental amount of time after each attempt, starting at
  127. wait_incrementing_start and incrementing by wait_incrementing_increment
  128. """
  129. result = self._wait_incrementing_start + (self._wait_incrementing_increment * (previous_attempt_number - 1))
  130. if result < 0:
  131. result = 0
  132. return result
  133. def exponential_sleep(self, previous_attempt_number, delay_since_first_attempt_ms):
  134. exp = 2 ** previous_attempt_number
  135. result = self._wait_exponential_multiplier * exp
  136. if result > self._wait_exponential_max:
  137. result = self._wait_exponential_max
  138. if result < 0:
  139. result = 0
  140. return result
  141. def never_reject(self, result):
  142. return False
  143. def always_reject(self, result):
  144. return True
  145. def should_reject(self, attempt):
  146. reject = False
  147. if attempt.has_exception:
  148. reject |= self._retry_on_exception(attempt.value[1])
  149. else:
  150. reject |= self._retry_on_result(attempt.value)
  151. return reject
  152. def call(self, fn, *args, **kwargs):
  153. start_time = int(round(time.time() * 1000))
  154. attempt_number = 1
  155. while True:
  156. try:
  157. attempt = Attempt(fn(*args, **kwargs), attempt_number, False)
  158. except:
  159. tb = sys.exc_info()
  160. attempt = Attempt(tb, attempt_number, True)
  161. if not self.should_reject(attempt):
  162. return attempt.get(self._wrap_exception)
  163. delay_since_first_attempt_ms = int(round(time.time() * 1000)) - start_time
  164. if self.stop(attempt_number, delay_since_first_attempt_ms):
  165. if not self._wrap_exception and attempt.has_exception:
  166. # get() on an attempt with an exception should cause it to be raised, but raise just in case
  167. raise attempt.get()
  168. else:
  169. raise RetryError(attempt)
  170. else:
  171. sleep = self.wait(attempt_number, delay_since_first_attempt_ms)
  172. if self._wait_jitter_max:
  173. jitter = random.random() * self._wait_jitter_max
  174. sleep = sleep + max(0, jitter)
  175. time.sleep(sleep / 1000.0)
  176. attempt_number += 1
  177. class Attempt(object):
  178. """
  179. An Attempt encapsulates a call to a target function that may end as a
  180. normal return value from the function or an Exception depending on what
  181. occurred during the execution.
  182. """
  183. def __init__(self, value, attempt_number, has_exception):
  184. self.value = value
  185. self.attempt_number = attempt_number
  186. self.has_exception = has_exception
  187. def get(self, wrap_exception=False):
  188. """
  189. Return the return value of this Attempt instance or raise an Exception.
  190. If wrap_exception is true, this Attempt is wrapped inside of a
  191. RetryError before being raised.
  192. """
  193. if self.has_exception:
  194. if wrap_exception:
  195. raise RetryError(self)
  196. else:
  197. six.reraise(self.value[0], self.value[1], self.value[2])
  198. else:
  199. return self.value
  200. def __repr__(self):
  201. if self.has_exception:
  202. return "Attempts: {0}, Error:\n{1}".format(self.attempt_number, "".join(traceback.format_tb(self.value[2])))
  203. else:
  204. return "Attempts: {0}, Value: {1}".format(self.attempt_number, self.value)
  205. class RetryError(Exception):
  206. """
  207. A RetryError encapsulates the last Attempt instance right before giving up.
  208. """
  209. def __init__(self, last_attempt):
  210. self.last_attempt = last_attempt
  211. def __str__(self):
  212. return "RetryError[{0}]".format(self.last_attempt)