1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
package agent
import (
"runtime"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/apimachinery/pkg/util/wait"
)
func runFlakyTest(t require.TestingT, flakyTest func(t require.TestingT)) {
totalAttempts := 3
var mockTestRunner *mockT
for i := 0; i < totalAttempts; i++ {
mockTestRunner = &mockT{}
var wg wait.Group
wg.Start(func() {
flakyTest(mockTestRunner)
})
wg.Wait()
if !mockTestRunner.failed {
return
}
}
if errorDetails := mockTestRunner.lastErrorDetails; errorDetails != nil {
t.Errorf(errorDetails.format, errorDetails.args...)
}
if mockTestRunner.failed {
t.FailNow()
}
}
// mockT is a test runner that implements require.TestingT interface. This is required because there
// is no way to run a test using *testing.T and ignore failures of subtests. However, assertions in the
// testify library do not explicitly require *testing.T but anything that implements require.TestingT interface.
// Hence, for scenarios where multiple attempts at testing are required, it is more convenient to use
// an instance of mockT and a test function that has require.TestingT in its function signature instead of *testing.T
type mockT struct {
failed bool
lastErrorDetails *mockTErrorDetails
}
type mockTErrorDetails struct {
format string
args []interface{}
}
func (t *mockT) FailNow() {
t.failed = true
runtime.Goexit()
}
func (t *mockT) Errorf(format string, args ...interface{}) {
t.lastErrorDetails = &mockTErrorDetails{
format: format,
args: args,
}
}
func TestFlakyTestRunner(t *testing.T) {
suite.Run(t, new(flakyTestRunnerSuite))
}
type flakyTestRunnerSuite struct {
suite.Suite
}
func (f *flakyTestRunnerSuite) TestSuccessInFirstAttempt() {
totalAttempts := 0
runFlakyTest(f.T(), func(t require.TestingT) {
totalAttempts++
})
f.EqualValues(1, totalAttempts)
}
func (f *flakyTestRunnerSuite) TestSuccessInLastAttempt() {
totalAttempts := 0
runFlakyTest(f.T(), func(t require.TestingT) {
totalAttempts++
if totalAttempts <= 2 {
t.FailNow()
}
})
f.EqualValues(3, totalAttempts)
}
func (f *flakyTestRunnerSuite) TestFailureInAllAttempts() {
totalAttempts := 0
mockTestRunner := &mockT{}
// calling this in a goroutine as the failure in the last attempt
// is expected to stop the goroutine following the convention of failed tests
// in Golang's testing package
var wg wait.Group
wg.Start(func() {
runFlakyTest(mockTestRunner, func(t require.TestingT) {
totalAttempts++
t.FailNow()
})
})
wg.Wait()
f.EqualValues(3, totalAttempts)
f.True(mockTestRunner.failed)
}
|
