1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
package aws
import (
"math/rand"
"net"
"net/http"
"time"
)
const (
maxDelay = 20 * time.Second
defaultScale = 300 * time.Millisecond
throttlingScale = 500 * time.Millisecond
throttlingScaleRange = throttlingScale / 4
defaultMaxRetries = 3
dynamoDBScale = 25 * time.Millisecond
dynamoDBMaxRetries = 10
)
// A RetryPolicy encapsulates a strategy for implementing client retries.
//
// Default implementations are provided which match the AWS SDKs.
type RetryPolicy interface {
// ShouldRetry returns whether a client should retry a failed request.
ShouldRetry(target string, r *http.Response, err error, numRetries int) bool
// Delay returns the time a client should wait before issuing a retry.
Delay(target string, r *http.Response, err error, numRetries int) time.Duration
}
// DefaultRetryPolicy implements the AWS SDK default retry policy.
//
// It will retry up to 3 times, and uses an exponential backoff with a scale
// factor of 300ms (300ms, 600ms, 1200ms). If the retry is because of
// throttling, the delay will also include some randomness.
//
// See https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/retry/PredefinedRetryPolicies.java#L90.
type DefaultRetryPolicy struct {
}
// ShouldRetry implements the RetryPolicy ShouldRetry method.
func (policy DefaultRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
return shouldRetry(r, err, numRetries, defaultMaxRetries)
}
// Delay implements the RetryPolicy Delay method.
func (policy DefaultRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
scale := defaultScale
if err, ok := err.(*Error); ok && isThrottlingException(err) {
scale = throttlingScale + time.Duration(rand.Int63n(int64(throttlingScaleRange)))
}
return exponentialBackoff(numRetries, scale)
}
// DynamoDBRetryPolicy implements the AWS SDK DynamoDB retry policy.
//
// It will retry up to 10 times, and uses an exponential backoff with a scale
// factor of 25ms (25ms, 50ms, 100ms, ...).
//
// See https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-core/src/main/java/com/amazonaws/retry/PredefinedRetryPolicies.java#L103.
type DynamoDBRetryPolicy struct {
}
// ShouldRetry implements the RetryPolicy ShouldRetry method.
func (policy DynamoDBRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
return shouldRetry(r, err, numRetries, dynamoDBMaxRetries)
}
// Delay implements the RetryPolicy Delay method.
func (policy DynamoDBRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
return exponentialBackoff(numRetries, dynamoDBScale)
}
// NeverRetryPolicy never retries requests and returns immediately on failure.
type NeverRetryPolicy struct {
}
// ShouldRetry implements the RetryPolicy ShouldRetry method.
func (policy NeverRetryPolicy) ShouldRetry(target string, r *http.Response, err error, numRetries int) bool {
return false
}
// Delay implements the RetryPolicy Delay method.
func (policy NeverRetryPolicy) Delay(target string, r *http.Response, err error, numRetries int) time.Duration {
return time.Duration(0)
}
// shouldRetry determines if we should retry the request.
//
// See http://docs.aws.amazon.com/general/latest/gr/api-retries.html.
func shouldRetry(r *http.Response, err error, numRetries int, maxRetries int) bool {
// Once we've exceeded the max retry attempts, game over.
if numRetries >= maxRetries {
return false
}
// Always retry temporary network errors.
if err, ok := err.(net.Error); ok && err.Temporary() {
return true
}
// Always retry 5xx responses.
if r != nil && r.StatusCode >= 500 {
return true
}
// Always retry throttling exceptions.
if err, ok := err.(ServiceError); ok && isThrottlingException(err) {
return true
}
// Other classes of failures indicate a problem with the request. Retrying
// won't help.
return false
}
func exponentialBackoff(numRetries int, scale time.Duration) time.Duration {
if numRetries < 0 {
return time.Duration(0)
}
delay := (1 << uint(numRetries)) * scale
if delay > maxDelay {
return maxDelay
}
return delay
}
func isThrottlingException(err ServiceError) bool {
switch err.ErrorCode() {
case "Throttling", "ThrottlingException", "ProvisionedThroughputExceededException":
return true
default:
return false
}
}
|
