File: flaky_test.go

package info (click to toggle)
gitlab-agent 16.11.5-1
  • links: PTS, VCS
  • area: contrib
  • in suites: experimental
  • size: 7,072 kB
  • sloc: makefile: 193; sh: 55; ruby: 3
file content (116 lines) | stat: -rw-r--r-- 2,635 bytes parent folder | download
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
package agent

import (
	"runtime"
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/stretchr/testify/suite"
	"k8s.io/apimachinery/pkg/util/wait"
)

func runFlakyTest(t require.TestingT, flakyTest func(t require.TestingT)) {
	totalAttempts := 3

	var mockTestRunner *mockT

	for i := 0; i < totalAttempts; i++ {
		mockTestRunner = &mockT{}

		var wg wait.Group
		wg.Start(func() {
			flakyTest(mockTestRunner)
		})

		wg.Wait()
		if !mockTestRunner.failed {
			return
		}
	}

	if errorDetails := mockTestRunner.lastErrorDetails; errorDetails != nil {
		t.Errorf(errorDetails.format, errorDetails.args...)
	}

	if mockTestRunner.failed {
		t.FailNow()
	}
}

// mockT is a test runner that implements require.TestingT interface. This is required because there
// is no way to run a test using *testing.T and ignore failures of subtests. However, assertions in the
// testify library do not explicitly require *testing.T but anything that implements require.TestingT interface.
// Hence, for scenarios where multiple attempts at testing are required, it is more convenient to use
// an instance of mockT and a test function that has require.TestingT in its function signature instead of *testing.T
type mockT struct {
	failed           bool
	lastErrorDetails *mockTErrorDetails
}

type mockTErrorDetails struct {
	format string
	args   []interface{}
}

func (t *mockT) FailNow() {
	t.failed = true
	runtime.Goexit()
}

func (t *mockT) Errorf(format string, args ...interface{}) {
	t.lastErrorDetails = &mockTErrorDetails{
		format: format,
		args:   args,
	}
}

func TestFlakyTestRunner(t *testing.T) {
	suite.Run(t, new(flakyTestRunnerSuite))
}

type flakyTestRunnerSuite struct {
	suite.Suite
}

func (f *flakyTestRunnerSuite) TestSuccessInFirstAttempt() {
	totalAttempts := 0

	runFlakyTest(f.T(), func(t require.TestingT) {
		totalAttempts++
	})

	f.EqualValues(1, totalAttempts)
}

func (f *flakyTestRunnerSuite) TestSuccessInLastAttempt() {
	totalAttempts := 0

	runFlakyTest(f.T(), func(t require.TestingT) {
		totalAttempts++
		if totalAttempts <= 2 {
			t.FailNow()
		}
	})

	f.EqualValues(3, totalAttempts)
}

func (f *flakyTestRunnerSuite) TestFailureInAllAttempts() {
	totalAttempts := 0
	mockTestRunner := &mockT{}

	// calling this in a goroutine as the failure in the last attempt
	// is expected to stop the goroutine following the convention of failed tests
	// in Golang's testing package
	var wg wait.Group
	wg.Start(func() {
		runFlakyTest(mockTestRunner, func(t require.TestingT) {
			totalAttempts++
			t.FailNow()
		})
	})
	wg.Wait()

	f.EqualValues(3, totalAttempts)
	f.True(mockTestRunner.failed)
}