1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
|
package v5
import (
"fmt"
"log"
"net/http"
"time"
"github.com/cenkalti/backoff"
)
// net/http RoundTripper interface, a.k.a. Transport
// https://godoc.org/net/http#RoundTripper
type RoundTripWithRetryBackoff struct {
// Configuration fields for backoff.ExponentialBackOff
InitialIntervalSeconds int64
RandomizationFactor float64
Multiplier float64
MaxIntervalSeconds int64
// After MaxElapsedTime the ExponentialBackOff stops.
// It never stops if MaxElapsedTime == 0.
MaxElapsedTimeSeconds int64
}
func (r RoundTripWithRetryBackoff) RoundTrip(req *http.Request) (*http.Response, error) {
var lastResponse *http.Response
var lastError error
retryableRoundTrip := func() error {
lastResponse = nil
lastError = nil
lastResponse, lastError = http.DefaultTransport.RoundTrip(req)
// Detect Heroku API rate limiting
// https://devcenter.heroku.com/articles/platform-api-reference#client-error-responses
if lastResponse != nil && lastResponse.StatusCode == 429 {
return fmt.Errorf("Heroku API rate limited: 429 Too Many Requests")
}
return nil
}
rateLimitRetryConfig := &backoff.ExponentialBackOff{
Clock: backoff.SystemClock,
InitialInterval: time.Duration(int64WithDefault(r.InitialIntervalSeconds, int64(30))) * time.Second,
RandomizationFactor: float64WithDefault(r.RandomizationFactor, float64(0.25)),
Multiplier: float64WithDefault(r.Multiplier, float64(2)),
MaxInterval: time.Duration(int64WithDefault(r.MaxIntervalSeconds, int64(900))) * time.Second,
MaxElapsedTime: time.Duration(int64WithDefault(r.MaxElapsedTimeSeconds, int64(0))) * time.Second,
}
rateLimitRetryConfig.Reset()
err := backoff.RetryNotify(retryableRoundTrip, rateLimitRetryConfig, notifyLog)
// Propagate the rate limit error when retries eventually fail.
if err != nil {
if lastResponse != nil {
lastResponse.Body.Close()
}
return nil, err
}
// Propagate all other response errors.
if lastError != nil {
if lastResponse != nil {
lastResponse.Body.Close()
}
return nil, lastError
}
return lastResponse, nil
}
func int64WithDefault(v int64, defaultV int64) int64 {
if v == int64(0) {
return defaultV
} else {
return v
}
}
func float64WithDefault(v float64, defaultV float64) float64 {
if v == float64(0) {
return defaultV
} else {
return v
}
}
func notifyLog(err error, waitDuration time.Duration) {
log.Printf("Will retry Heroku API request in %s, because %s", waitDuration, err)
}
|
