1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269
|
package retry
import (
"context"
"fmt"
"time"
"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
)
// BackoffDelayer provides the interface for determining the delay to before
// another request attempt, that previously failed.
type BackoffDelayer interface {
BackoffDelay(attempt int, err error) (time.Duration, error)
}
// BackoffDelayerFunc provides a wrapper around a function to determine the
// backoff delay of an attempt retry.
type BackoffDelayerFunc func(int, error) (time.Duration, error)
// BackoffDelay returns the delay before attempt to retry a request.
func (fn BackoffDelayerFunc) BackoffDelay(attempt int, err error) (time.Duration, error) {
return fn(attempt, err)
}
const (
// DefaultMaxAttempts is the maximum of attempts for an API request
DefaultMaxAttempts int = 3
// DefaultMaxBackoff is the maximum back off delay between attempts
DefaultMaxBackoff time.Duration = 20 * time.Second
)
// Default retry token quota values.
const (
DefaultRetryRateTokens uint = 500
DefaultRetryCost uint = 5
DefaultRetryTimeoutCost uint = 10
DefaultNoRetryIncrement uint = 1
)
// DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK
// should consider as retryable errors.
var DefaultRetryableHTTPStatusCodes = map[int]struct{}{
500: {},
502: {},
503: {},
504: {},
}
// DefaultRetryableErrorCodes provides the set of API error codes that should
// be retried.
var DefaultRetryableErrorCodes = map[string]struct{}{
"RequestTimeout": {},
"RequestTimeoutException": {},
}
// DefaultThrottleErrorCodes provides the set of API error codes that are
// considered throttle errors.
var DefaultThrottleErrorCodes = map[string]struct{}{
"Throttling": {},
"ThrottlingException": {},
"ThrottledException": {},
"RequestThrottledException": {},
"TooManyRequestsException": {},
"ProvisionedThroughputExceededException": {},
"TransactionInProgressException": {},
"RequestLimitExceeded": {},
"BandwidthLimitExceeded": {},
"LimitExceededException": {},
"RequestThrottled": {},
"SlowDown": {},
"PriorRequestNotComplete": {},
"EC2ThrottledException": {},
}
// DefaultRetryables provides the set of retryable checks that are used by
// default.
var DefaultRetryables = []IsErrorRetryable{
NoRetryCanceledError{},
RetryableError{},
RetryableConnectionError{},
RetryableHTTPStatusCode{
Codes: DefaultRetryableHTTPStatusCodes,
},
RetryableErrorCode{
Codes: DefaultRetryableErrorCodes,
},
RetryableErrorCode{
Codes: DefaultThrottleErrorCodes,
},
}
// DefaultTimeouts provides the set of timeout checks that are used by default.
var DefaultTimeouts = []IsErrorTimeout{
TimeouterError{},
}
// StandardOptions provides the functional options for configuring the standard
// retryable, and delay behavior.
type StandardOptions struct {
// Maximum number of attempts that should be made.
MaxAttempts int
// MaxBackoff duration between retried attempts.
MaxBackoff time.Duration
// Provides the backoff strategy the retryer will use to determine the
// delay between retry attempts.
Backoff BackoffDelayer
// Set of strategies to determine if the attempt should be retried based on
// the error response received.
//
// It is safe to append to this list in NewStandard's functional options.
Retryables []IsErrorRetryable
// Set of strategies to determine if the attempt failed due to a timeout
// error.
//
// It is safe to append to this list in NewStandard's functional options.
Timeouts []IsErrorTimeout
// Provides the rate limiting strategy for rate limiting attempt retries
// across all attempts the retryer is being used with.
//
// A RateLimiter operates as a token bucket with a set capacity, where
// attempt failures events consume tokens. A retry attempt that attempts to
// consume more tokens than what's available results in operation failure.
// The default implementation is parameterized as follows:
// - a capacity of 500 (DefaultRetryRateTokens)
// - a retry caused by a timeout costs 10 tokens (DefaultRetryCost)
// - a retry caused by other errors costs 5 tokens (DefaultRetryTimeoutCost)
// - an operation that succeeds on the 1st attempt adds 1 token (DefaultNoRetryIncrement)
//
// You can disable rate limiting by setting this field to ratelimit.None.
RateLimiter RateLimiter
// The cost to deduct from the RateLimiter's token bucket per retry.
RetryCost uint
// The cost to deduct from the RateLimiter's token bucket per retry caused
// by timeout error.
RetryTimeoutCost uint
// The cost to payback to the RateLimiter's token bucket for successful
// attempts.
NoRetryIncrement uint
}
// RateLimiter provides the interface for limiting the rate of attempt retries
// allowed by the retryer.
type RateLimiter interface {
GetToken(ctx context.Context, cost uint) (releaseToken func() error, err error)
AddTokens(uint) error
}
// Standard is the standard retry pattern for the SDK. It uses a set of
// retryable checks to determine of the failed attempt should be retried, and
// what retry delay should be used.
type Standard struct {
options StandardOptions
timeout IsErrorTimeout
retryable IsErrorRetryable
backoff BackoffDelayer
}
// NewStandard initializes a standard retry behavior with defaults that can be
// overridden via functional options.
func NewStandard(fnOpts ...func(*StandardOptions)) *Standard {
o := StandardOptions{
MaxAttempts: DefaultMaxAttempts,
MaxBackoff: DefaultMaxBackoff,
Retryables: append([]IsErrorRetryable{}, DefaultRetryables...),
Timeouts: append([]IsErrorTimeout{}, DefaultTimeouts...),
RateLimiter: ratelimit.NewTokenRateLimit(DefaultRetryRateTokens),
RetryCost: DefaultRetryCost,
RetryTimeoutCost: DefaultRetryTimeoutCost,
NoRetryIncrement: DefaultNoRetryIncrement,
}
for _, fn := range fnOpts {
fn(&o)
}
if o.MaxAttempts <= 0 {
o.MaxAttempts = DefaultMaxAttempts
}
backoff := o.Backoff
if backoff == nil {
backoff = NewExponentialJitterBackoff(o.MaxBackoff)
}
return &Standard{
options: o,
backoff: backoff,
retryable: IsErrorRetryables(o.Retryables),
timeout: IsErrorTimeouts(o.Timeouts),
}
}
// MaxAttempts returns the maximum number of attempts that can be made for a
// request before failing.
func (s *Standard) MaxAttempts() int {
return s.options.MaxAttempts
}
// IsErrorRetryable returns if the error is can be retried or not. Should not
// consider the number of attempts made.
func (s *Standard) IsErrorRetryable(err error) bool {
return s.retryable.IsErrorRetryable(err).Bool()
}
// RetryDelay returns the delay to use before another request attempt is made.
func (s *Standard) RetryDelay(attempt int, err error) (time.Duration, error) {
return s.backoff.BackoffDelay(attempt, err)
}
// GetAttemptToken returns the token to be released after then attempt completes.
// The release token will add NoRetryIncrement to the RateLimiter token pool if
// the attempt was successful. If the attempt failed, nothing will be done.
func (s *Standard) GetAttemptToken(context.Context) (func(error) error, error) {
return s.GetInitialToken(), nil
}
// GetInitialToken returns a token for adding the NoRetryIncrement to the
// RateLimiter token if the attempt completed successfully without error.
//
// InitialToken applies to result of the each attempt, including the first.
// Whereas the RetryToken applies to the result of subsequent attempts.
//
// Deprecated: use GetAttemptToken instead.
func (s *Standard) GetInitialToken() func(error) error {
return releaseToken(s.noRetryIncrement).release
}
func (s *Standard) noRetryIncrement() error {
return s.options.RateLimiter.AddTokens(s.options.NoRetryIncrement)
}
// GetRetryToken attempts to deduct the retry cost from the retry token pool.
// Returning the token release function, or error.
func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) {
cost := s.options.RetryCost
if s.timeout.IsErrorTimeout(opErr).Bool() {
cost = s.options.RetryTimeoutCost
}
fn, err := s.options.RateLimiter.GetToken(ctx, cost)
if err != nil {
return nil, fmt.Errorf("failed to get rate limit token, %w", err)
}
return releaseToken(fn).release, nil
}
func nopRelease(error) error { return nil }
type releaseToken func() error
func (f releaseToken) release(err error) error {
if err != nil {
return nil
}
return f()
}
|