File: standard.go

package info (click to toggle)
golang-github-aws-aws-sdk-go-v2 1.30.3-1
  • links: PTS, VCS
  • area: main
  • in suites: forky, sid, trixie
  • size: 662,428 kB
  • sloc: java: 16,875; makefile: 432; sh: 175
file content (269 lines) | stat: -rw-r--r-- 8,642 bytes parent folder | download | duplicates (5)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
package retry

import (
	"context"
	"fmt"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws/ratelimit"
)

// BackoffDelayer provides the interface for determining the delay to before
// another request attempt, that previously failed.
type BackoffDelayer interface {
	BackoffDelay(attempt int, err error) (time.Duration, error)
}

// BackoffDelayerFunc provides a wrapper around a function to determine the
// backoff delay of an attempt retry.
type BackoffDelayerFunc func(int, error) (time.Duration, error)

// BackoffDelay returns the delay before attempt to retry a request.
func (fn BackoffDelayerFunc) BackoffDelay(attempt int, err error) (time.Duration, error) {
	return fn(attempt, err)
}

const (
	// DefaultMaxAttempts is the maximum of attempts for an API request
	DefaultMaxAttempts int = 3

	// DefaultMaxBackoff is the maximum back off delay between attempts
	DefaultMaxBackoff time.Duration = 20 * time.Second
)

// Default retry token quota values.
const (
	DefaultRetryRateTokens  uint = 500
	DefaultRetryCost        uint = 5
	DefaultRetryTimeoutCost uint = 10
	DefaultNoRetryIncrement uint = 1
)

// DefaultRetryableHTTPStatusCodes is the default set of HTTP status codes the SDK
// should consider as retryable errors.
var DefaultRetryableHTTPStatusCodes = map[int]struct{}{
	500: {},
	502: {},
	503: {},
	504: {},
}

// DefaultRetryableErrorCodes provides the set of API error codes that should
// be retried.
var DefaultRetryableErrorCodes = map[string]struct{}{
	"RequestTimeout":          {},
	"RequestTimeoutException": {},
}

// DefaultThrottleErrorCodes provides the set of API error codes that are
// considered throttle errors.
var DefaultThrottleErrorCodes = map[string]struct{}{
	"Throttling":                             {},
	"ThrottlingException":                    {},
	"ThrottledException":                     {},
	"RequestThrottledException":              {},
	"TooManyRequestsException":               {},
	"ProvisionedThroughputExceededException": {},
	"TransactionInProgressException":         {},
	"RequestLimitExceeded":                   {},
	"BandwidthLimitExceeded":                 {},
	"LimitExceededException":                 {},
	"RequestThrottled":                       {},
	"SlowDown":                               {},
	"PriorRequestNotComplete":                {},
	"EC2ThrottledException":                  {},
}

// DefaultRetryables provides the set of retryable checks that are used by
// default.
var DefaultRetryables = []IsErrorRetryable{
	NoRetryCanceledError{},
	RetryableError{},
	RetryableConnectionError{},
	RetryableHTTPStatusCode{
		Codes: DefaultRetryableHTTPStatusCodes,
	},
	RetryableErrorCode{
		Codes: DefaultRetryableErrorCodes,
	},
	RetryableErrorCode{
		Codes: DefaultThrottleErrorCodes,
	},
}

// DefaultTimeouts provides the set of timeout checks that are used by default.
var DefaultTimeouts = []IsErrorTimeout{
	TimeouterError{},
}

// StandardOptions provides the functional options for configuring the standard
// retryable, and delay behavior.
type StandardOptions struct {
	// Maximum number of attempts that should be made.
	MaxAttempts int

	// MaxBackoff duration between retried attempts.
	MaxBackoff time.Duration

	// Provides the backoff strategy the retryer will use to determine the
	// delay between retry attempts.
	Backoff BackoffDelayer

	// Set of strategies to determine if the attempt should be retried based on
	// the error response received.
	//
	// It is safe to append to this list in NewStandard's functional options.
	Retryables []IsErrorRetryable

	// Set of strategies to determine if the attempt failed due to a timeout
	// error.
	//
	// It is safe to append to this list in NewStandard's functional options.
	Timeouts []IsErrorTimeout

	// Provides the rate limiting strategy for rate limiting attempt retries
	// across all attempts the retryer is being used with.
	//
	// A RateLimiter operates as a token bucket with a set capacity, where
	// attempt failures events consume tokens. A retry attempt that attempts to
	// consume more tokens than what's available results in operation failure.
	// The default implementation is parameterized as follows:
	//   - a capacity of 500 (DefaultRetryRateTokens)
	//   - a retry caused by a timeout costs 10 tokens (DefaultRetryCost)
	//   - a retry caused by other errors costs 5 tokens (DefaultRetryTimeoutCost)
	//   - an operation that succeeds on the 1st attempt adds 1 token (DefaultNoRetryIncrement)
	//
	// You can disable rate limiting by setting this field to ratelimit.None.
	RateLimiter RateLimiter

	// The cost to deduct from the RateLimiter's token bucket per retry.
	RetryCost uint

	// The cost to deduct from the RateLimiter's token bucket per retry caused
	// by timeout error.
	RetryTimeoutCost uint

	// The cost to payback to the RateLimiter's token bucket for successful
	// attempts.
	NoRetryIncrement uint
}

// RateLimiter provides the interface for limiting the rate of attempt retries
// allowed by the retryer.
type RateLimiter interface {
	GetToken(ctx context.Context, cost uint) (releaseToken func() error, err error)
	AddTokens(uint) error
}

// Standard is the standard retry pattern for the SDK. It uses a set of
// retryable checks to determine of the failed attempt should be retried, and
// what retry delay should be used.
type Standard struct {
	options StandardOptions

	timeout   IsErrorTimeout
	retryable IsErrorRetryable
	backoff   BackoffDelayer
}

// NewStandard initializes a standard retry behavior with defaults that can be
// overridden via functional options.
func NewStandard(fnOpts ...func(*StandardOptions)) *Standard {
	o := StandardOptions{
		MaxAttempts: DefaultMaxAttempts,
		MaxBackoff:  DefaultMaxBackoff,
		Retryables:  append([]IsErrorRetryable{}, DefaultRetryables...),
		Timeouts:    append([]IsErrorTimeout{}, DefaultTimeouts...),

		RateLimiter:      ratelimit.NewTokenRateLimit(DefaultRetryRateTokens),
		RetryCost:        DefaultRetryCost,
		RetryTimeoutCost: DefaultRetryTimeoutCost,
		NoRetryIncrement: DefaultNoRetryIncrement,
	}
	for _, fn := range fnOpts {
		fn(&o)
	}
	if o.MaxAttempts <= 0 {
		o.MaxAttempts = DefaultMaxAttempts
	}

	backoff := o.Backoff
	if backoff == nil {
		backoff = NewExponentialJitterBackoff(o.MaxBackoff)
	}

	return &Standard{
		options:   o,
		backoff:   backoff,
		retryable: IsErrorRetryables(o.Retryables),
		timeout:   IsErrorTimeouts(o.Timeouts),
	}
}

// MaxAttempts returns the maximum number of attempts that can be made for a
// request before failing.
func (s *Standard) MaxAttempts() int {
	return s.options.MaxAttempts
}

// IsErrorRetryable returns if the error is can be retried or not. Should not
// consider the number of attempts made.
func (s *Standard) IsErrorRetryable(err error) bool {
	return s.retryable.IsErrorRetryable(err).Bool()
}

// RetryDelay returns the delay to use before another request attempt is made.
func (s *Standard) RetryDelay(attempt int, err error) (time.Duration, error) {
	return s.backoff.BackoffDelay(attempt, err)
}

// GetAttemptToken returns the token to be released after then attempt completes.
// The release token will add NoRetryIncrement to the RateLimiter token pool if
// the attempt was successful. If the attempt failed, nothing will be done.
func (s *Standard) GetAttemptToken(context.Context) (func(error) error, error) {
	return s.GetInitialToken(), nil
}

// GetInitialToken returns a token for adding the NoRetryIncrement to the
// RateLimiter token if the attempt completed successfully without error.
//
// InitialToken applies to result of the each attempt, including the first.
// Whereas the RetryToken applies to the result of subsequent attempts.
//
// Deprecated: use GetAttemptToken instead.
func (s *Standard) GetInitialToken() func(error) error {
	return releaseToken(s.noRetryIncrement).release
}

func (s *Standard) noRetryIncrement() error {
	return s.options.RateLimiter.AddTokens(s.options.NoRetryIncrement)
}

// GetRetryToken attempts to deduct the retry cost from the retry token pool.
// Returning the token release function, or error.
func (s *Standard) GetRetryToken(ctx context.Context, opErr error) (func(error) error, error) {
	cost := s.options.RetryCost

	if s.timeout.IsErrorTimeout(opErr).Bool() {
		cost = s.options.RetryTimeoutCost
	}

	fn, err := s.options.RateLimiter.GetToken(ctx, cost)
	if err != nil {
		return nil, fmt.Errorf("failed to get rate limit token, %w", err)
	}

	return releaseToken(fn).release, nil
}

func nopRelease(error) error { return nil }

type releaseToken func() error

func (f releaseToken) release(err error) error {
	if err != nil {
		return nil
	}

	return f()
}