1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
|
# frozen_string_literal: true
require 'test_helper'
class StatsFromDirTest < ActiveSupport::TestCase
test "that it works" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
assert newest.average < oldest.average
assert_equal "winner", newest.name
assert_equal "loser", oldest.name
assert_in_delta 0.26, stats.d_max, 0.01
assert_in_delta 0.2145966026289347, stats.d_critical, 0.00001
assert_equal true, stats.significant?
format = DerailedBenchmarks::StatsFromDir::FORMAT
assert_equal "1.0062", format % stats.x_faster
assert_equal "0.6147", format % stats.percent_faster
assert_equal "11.3844", format % newest.median
end
test "histogram output" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
io = StringIO.new
stats.call.banner(io)
puts io.string
assert_match(/11\.2 , 11\.28/, io.string)
assert_match(/11\.8 , 11\.88/, io.string)
end
test "alignment" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
def stats.percent_faster
-0.1
end
def stats.x_faster
0.9922
end
assert_equal 1, stats.align.length
end
test "banner faster" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
# Test fixture for banner
def stats.d_max
"0.037"
end
def stats.d_critical
"0.001"
end
def newest.median
10.5
end
def oldest.median
11.0
end
expected = <<~EOM
[winner] "I am the new commit" - (10.5 seconds)
FASTER 🚀🚀🚀 by:
1.0476x [older/newer]
4.5455% [(older - newer) / older * 100]
[loser] "Old commit" - (11.0 seconds)
EOM
actual = StringIO.new
stats.banner(actual)
assert_match expected, actual.string
end
test "banner slower" do
dir = fixtures_dir("stats/significant")
branch_info = {}
branch_info["loser"] = { desc: "I am the new commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
branch_info["winner"] = { desc: "Old commit", time: Time.now - 10, file: dir.join("winner.bench.txt"), name: "winner" }
stats = DerailedBenchmarks::StatsFromDir.new(branch_info).call
newest = stats.newest
oldest = stats.oldest
def oldest.median
10.5
end
def newest.median
11.0
end
expected = <<~EOM
[loser] "I am the new commit" - (11.0 seconds)
SLOWER 🐢🐢🐢 by:
0.9545x [older/newer]
-4.7619% [(older - newer) / older * 100]
[winner] "Old commit" - (10.5 seconds)
EOM
actual = StringIO.new
stats.banner(actual)
assert_match expected, actual.string
end
test "stats from samples with slightly different sizes" do
stats = DerailedBenchmarks::StatsFromDir.new({})
out = stats.statistical_test([100,101,102, 100, 101, 99], [1,3, 3, 2])
assert out[:alternative]
end
end
|
