1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195
|
// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package testutil
import (
"fmt"
"net/http"
"os"
"regexp"
"runtime"
"sort"
"strings"
"testing"
"time"
)
// leakDetectionDisabled returns true if leak detection should be skipped.
// Set ETCD_TEST_NO_LEAK_DETECTION=1 to disable leak detection, which is
// useful in environments where the leak detector may produce false positives
// (e.g., Debian package builds with Go modules disabled).
func leakDetectionDisabled() bool {
return os.Getenv("ETCD_TEST_NO_LEAK_DETECTION") != ""
}
// TODO: Replace with https://github.com/uber-go/goleak.
/*
CheckLeakedGoroutine verifies tests do not leave any leaky
goroutines. It returns true when there are goroutines still
running(leaking) after all tests.
import "go.etcd.io/etcd/client/pkg/v3/testutil"
func TestMain(m *testing.M) {
testutil.MustTestMainWithLeakDetection(m)
}
func TestSample(t *testing.T) {
RegisterLeakDetection(t)
...
}
*/
func CheckLeakedGoroutine() bool {
gs := interestingGoroutines()
if len(gs) == 0 {
return false
}
stackCount := make(map[string]int)
re := regexp.MustCompile(`\(0[0-9a-fx, ]*\)`)
for _, g := range gs {
// strip out pointer arguments in first function of stack dump
normalized := string(re.ReplaceAll([]byte(g), []byte("(...)")))
stackCount[normalized]++
}
fmt.Fprintf(os.Stderr, "Unexpected goroutines running after all test(s).\n")
for stack, count := range stackCount {
fmt.Fprintf(os.Stderr, "%d instances of:\n%s\n", count, stack)
}
return true
}
// CheckAfterTest returns an error if AfterTest would fail with an error.
// Waits for go-routines shutdown for 'd'.
func CheckAfterTest(d time.Duration) error {
http.DefaultTransport.(*http.Transport).CloseIdleConnections()
var bad string
// Presence of these goroutines causes immediate test failure.
badSubstring := map[string]string{
").writeLoop(": "a Transport",
"created by net/http/httptest.(*Server).Start": "an httptest.Server",
"timeoutHandler": "a TimeoutHandler",
"net.(*netFD).connect(": "a timing out dial",
").noteClientGone(": "a closenotifier sender",
").readLoop(": "a Transport",
".grpc": "a gRPC resource",
").sendCloseSubstream(": "a stream closing routine",
}
var stacks string
begin := time.Now()
for time.Since(begin) < d {
bad = ""
goroutines := interestingGoroutines()
if len(goroutines) == 0 {
return nil
}
stacks = strings.Join(goroutines, "\n\n")
for substr, what := range badSubstring {
if strings.Contains(stacks, substr) {
bad = what
}
}
// Undesired goroutines found, but goroutines might just still be
// shutting down, so give it some time.
runtime.Gosched()
time.Sleep(50 * time.Millisecond)
}
return fmt.Errorf("appears to have leaked %s:\n%s", bad, stacks)
}
// RegisterLeakDetection is a convenient way to register before-and-after code to a test.
// If you execute RegisterLeakDetection, you don't need to explicitly register AfterTest.
func RegisterLeakDetection(t TB) {
if leakDetectionDisabled() {
return
}
if err := CheckAfterTest(10 * time.Millisecond); err != nil {
t.Skip("Found leaked goroutined BEFORE test", err)
return
}
t.Cleanup(func() {
afterTest(t)
})
}
// afterTest is meant to run in a defer that executes after a test completes.
// It will detect common goroutine leaks, retrying in case there are goroutines
// not synchronously torn down, and fail the test if any goroutines are stuck.
func afterTest(t TB) {
// If test-failed the leaked goroutines list is hidding the real
// source of problem.
if !t.Failed() {
if err := CheckAfterTest(1 * time.Second); err != nil {
t.Errorf("Test %v", err)
}
}
}
func interestingGoroutines() (gs []string) {
buf := make([]byte, 2<<20)
buf = buf[:runtime.Stack(buf, true)]
for _, g := range strings.Split(string(buf), "\n\n") {
sl := strings.SplitN(g, "\n", 2)
if len(sl) != 2 {
continue
}
stack := strings.TrimSpace(sl[1])
if stack == "" ||
strings.Contains(stack, "sync.(*WaitGroup).Done") ||
strings.Contains(stack, "os.(*file).close") ||
strings.Contains(stack, "os.(*Process).Release") ||
strings.Contains(stack, "created by os/signal.init") ||
strings.Contains(stack, "runtime/panic.go") ||
strings.Contains(stack, "created by testing.RunTests") ||
strings.Contains(stack, "created by testing.runTests") ||
strings.Contains(stack, "created by testing.(*T).Run") ||
strings.Contains(stack, "testing.Main(") ||
strings.Contains(stack, "runtime.goexit") ||
strings.Contains(stack, "go.etcd.io/etcd/client/pkg/v3/testutil.interestingGoroutines") ||
strings.Contains(stack, "go.etcd.io/etcd/client/pkg/v3/logutil.(*MergeLogger).outputLoop") ||
strings.Contains(stack, "go.etcd.io/etcd/client/pkg/testutil.interestingGoroutines") ||
strings.Contains(stack, "go.etcd.io/etcd/client/pkg/logutil.(*MergeLogger).outputLoop") ||
strings.Contains(stack, "github.com/golang/glog.(*loggingT).flushDaemon") ||
strings.Contains(stack, "created by runtime.gc") ||
strings.Contains(stack, "created by text/template/parse.lex") ||
strings.Contains(stack, "runtime.MHeap_Scavenger") ||
strings.Contains(stack, "rcrypto/internal/boring.(*PublicKeyRSA).finalize") ||
strings.Contains(stack, "net.(*netFD).Close(") ||
strings.Contains(stack, "testing.(*T).Run") ||
strings.Contains(stack, "crypto/tls.(*certCache).evict") {
continue
}
gs = append(gs, stack)
}
sort.Strings(gs)
return gs
}
func MustCheckLeakedGoroutine() {
http.DefaultTransport.(*http.Transport).CloseIdleConnections()
CheckAfterTest(5 * time.Second)
// Let the other goroutines finalize.
runtime.Gosched()
if CheckLeakedGoroutine() {
os.Exit(1)
}
}
// MustTestMainWithLeakDetection expands standard m.Run with leaked
// goroutines detection. Set ETCD_TEST_NO_LEAK_DETECTION=1 to disable.
func MustTestMainWithLeakDetection(m *testing.M) {
v := m.Run()
if v == 0 && !leakDetectionDisabled() {
MustCheckLeakedGoroutine()
}
os.Exit(v)
}
|