1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
|
"""Acceptance test utils"""
import functools
import logging
import time
import traceback
import pika.compat
def retry_assertion(timeout_sec, retry_interval_sec=0.1):
"""Creates a decorator that retries the decorated function or
method only upon `AssertionError` exception at the given retry interval
not to exceed the overall given timeout.
:param float timeout_sec: overall timeout in seconds
:param float retry_interval_sec: amount of time to sleep
between retries in seconds.
:returns: decorator that implements the following behavior
1. This decorator guarantees to call the decorated function or method at
least once.
2. It passes through all exceptions besides `AssertionError`, preserving the
original exception and its traceback.
3. If no exception, it returns the return value from the decorated function/method.
4. It sleeps `time.sleep(retry_interval_sec)` between retries.
5. It checks for expiry of the overall timeout before sleeping.
6. If the overall timeout is exceeded, it re-raises the latest `AssertionError`,
preserving its original traceback
"""
def retry_assertion_decorator(func):
"""Decorator"""
@functools.wraps(func)
def retry_assertion_wrap(*args, **kwargs):
"""The wrapper"""
num_attempts = 0
start_time = pika.compat.time_now()
while True:
num_attempts += 1
try:
result = func(*args, **kwargs)
except AssertionError:
now = pika.compat.time_now()
# Compensate for time adjustment
if now < start_time:
start_time = now
if (now - start_time) > timeout_sec:
logging.exception(
'Exceeded retry timeout of %s sec in %s attempts '
'with func %r. Caller\'s stack:\n%s',
timeout_sec, num_attempts, func,
''.join(traceback.format_stack()))
raise
logging.debug('Attempt %s failed; retrying %r in %s sec.',
num_attempts, func, retry_interval_sec)
time.sleep(retry_interval_sec)
else:
logging.debug('%r succeeded at attempt %s',
func, num_attempts)
return result
return retry_assertion_wrap
return retry_assertion_decorator
|
