1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
// Copyright 2020 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package utils
import (
"fmt"
"strings"
"time"
)
// RetryUntil when the when func returns true
func RetryUntil(f func() error, when func(error) bool) error {
e := f()
if e == nil {
return nil
}
if when == nil {
return RetryUntil(f, nil)
} else if when(e) {
return RetryUntil(f, when)
}
return e
}
// RetryOption is options for Retry()
type RetryOption struct {
Attempts int64
Delay time.Duration
Timeout time.Duration
}
// default values for RetryOption
var (
defaultAttempts int64 = 20
defaultDelay = time.Millisecond * 500 // 500ms
defaultTimeout = time.Second * 10 // 10s
)
// Retry retries the func until it returns no error or reaches attempts limit or
// timed out, either one is earlier
func Retry(doFunc func() error, opts ...RetryOption) error {
var cfg RetryOption
if len(opts) > 0 {
cfg = opts[0]
} else {
cfg = RetryOption{
Attempts: defaultAttempts,
Delay: defaultDelay,
Timeout: defaultTimeout,
}
}
// timeout must be greater than 0
if cfg.Timeout <= 0 {
return fmt.Errorf("timeout (%s) must be greater than 0", cfg.Timeout)
}
// set options automatically for invalid value
if cfg.Delay <= 0 {
cfg.Delay = defaultDelay
}
if cfg.Attempts <= 0 {
cfg.Attempts = cfg.Timeout.Milliseconds()/cfg.Delay.Milliseconds() + 1
}
timeoutChan := time.After(cfg.Timeout)
// call the function
var attemptCount int64
var err error
for attemptCount = 0; attemptCount < cfg.Attempts; attemptCount++ {
if err = doFunc(); err == nil {
return nil
}
// check for timeout
select {
case <-timeoutChan:
return fmt.Errorf("operation timed out after %s", cfg.Timeout)
default:
time.Sleep(cfg.Delay)
}
}
return fmt.Errorf("operation exceeds the max retry attempts of %d. error of last attempt: %s", cfg.Attempts, err)
}
// IsTimeoutOrMaxRetry return true if it's timeout or reach max retry.
func IsTimeoutOrMaxRetry(err error) bool {
if err == nil {
return false
}
s := err.Error()
if strings.Contains(s, "operation timed out after") ||
strings.Contains(s, "operation exceeds the max retry attempts of") {
return true
}
return false
}
|
