1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116
|
package agent
import (
"runtime"
"testing"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"k8s.io/apimachinery/pkg/util/wait"
)
func runFlakyTest(t require.TestingT, flakyTest func(t require.TestingT)) {
totalAttempts := 3
var mockTestRunner *mockT
for i := 0; i < totalAttempts; i++ {
mockTestRunner = &mockT{}
var wg wait.Group
wg.Start(func() {
flakyTest(mockTestRunner)
})
wg.Wait()
if !mockTestRunner.failed {
return
}
}
if errorDetails := mockTestRunner.lastErrorDetails; errorDetails != nil {
t.Errorf(errorDetails.format, errorDetails.args...)
}
if mockTestRunner.failed {
t.FailNow()
}
}
// mockT is a test runner that implements require.TestingT interface. This is required because there
// is no way to run a test using *testing.T and ignore failures of subtests. However, assertions in the
// testify library do not explicitly require *testing.T but anything that implements require.TestingT interface.
// Hence, for scenarios where multiple attempts at testing are required, it is more convenient to use
// an instance of mockT and a test function that has require.TestingT in its function signature instead of *testing.T
type mockT struct {
failed bool
lastErrorDetails *mockTErrorDetails
}
type mockTErrorDetails struct {
format string
args []interface{}
}
func (t *mockT) FailNow() {
t.failed = true
runtime.Goexit()
}
func (t *mockT) Errorf(format string, args ...interface{}) {
t.lastErrorDetails = &mockTErrorDetails{
format: format,
args: args,
}
}
func TestFlakyTestRunner(t *testing.T) {
suite.Run(t, new(flakyTestRunnerSuite))
}
type flakyTestRunnerSuite struct {
suite.Suite
}
func (f *flakyTestRunnerSuite) TestSuccessInFirstAttempt() {
totalAttempts := 0
runFlakyTest(f.T(), func(t require.TestingT) {
totalAttempts++
})
f.EqualValues(1, totalAttempts)
}
func (f *flakyTestRunnerSuite) TestSuccessInLastAttempt() {
totalAttempts := 0
runFlakyTest(f.T(), func(t require.TestingT) {
totalAttempts++
if totalAttempts <= 2 {
t.FailNow()
}
})
f.EqualValues(3, totalAttempts)
}
func (f *flakyTestRunnerSuite) TestFailureInAllAttempts() {
totalAttempts := 0
mockTestRunner := &mockT{}
// calling this in a goroutine as the failure in the last attempt
// is expected to stop the goroutine following the convention of failed tests
// in Golang's testing package
var wg wait.Group
wg.Start(func() {
runFlakyTest(mockTestRunner, func(t require.TestingT) {
totalAttempts++
t.FailNow()
})
})
wg.Wait()
f.EqualValues(3, totalAttempts)
f.True(mockTestRunner.failed)
}
|