1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
|
package main
import (
"fmt"
"os"
"testing"
"github.com/johnkerl/miller/v6/pkg/terminals/regtest"
)
// TestRegression is a familiar entry point for regression testing. Miller
// regression tests are more flexibly invoked via 'mlr regtest'. However here
// is a standard location so people can get at them via 'go test'. Please see
// (as of this writing) pkg/terminals/regtest for the Miller regtest package.
func TestRegression(t *testing.T) {
// How much detail to show? There are thousands of cases, organized into a
// few hundred top-level directories under ./test/cases.
//
// Default behavior is to show PASS/FAIL for those top-level directories.
// If (for whatever reason) lots of tests are systematically failing then
// verbosityLevel = 3 for all cases is probably too much output to be
// useful.
//
// Also note our regtest framework supports four verbosity levels, 'mlr
// regtest' (0) through 'mlr regtest -vvv' (3), while 'go test' has only
// 'go test' and 'go test -v'. Our regtest framework also has 'mlr regtest
// -s 20' which means *re-run* up to 20 failing tests (after having failed
// once with verbosityLevel = 0) as if those had been invoked with
// verbosityLevel = 3.
//
// What we do is:
// * go test: like 'mlr regtest'
// * go test -v: like 'mlr regtest -s 20'
//
// This is (I hope) sufficient flexibility for use in GitHub Actions
// continuous-integration jobs. If more detail is needed then one may:
//
// * For CI debugging: simply edit the below parameters verbosityLevel
// and firstNFailsToShow and re-push to GitHub.
// * For interactive debug: run 'mlr regtest -v', 'mlr regtest -vv', 'mlr
// regtest -vvv' instead of going through 'go test'.
firstNFailsToShow := 0
if testing.Verbose() {
firstNFailsToShow = 20
}
// Let the tests find ./mlr
cwd, err := os.Getwd()
if err != nil {
fmt.Fprintln(os.Stderr, "mlr: could not find current working directory.")
os.Exit(1)
}
path := os.Getenv("PATH")
os.Setenv("PATH", cwd+":"+path)
// With 'go test' invoked from the repo base directory, the cwd for this code will
// be the tests/ subdirectory.
casePaths := []string{"./test/cases"} // use default
regtester := regtest.NewRegTester(
"mlr", // exeName
false, // doPopulate
0, // verbosityLevel
false, // plainMode
firstNFailsToShow,
)
ok := regtester.Execute(casePaths)
if !ok {
t.Fatal()
}
}
|
