1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
|
# frozen_string_literal: true
require 'bigdecimal'
require 'statistics'
require 'unicode_plot'
require 'stringio'
require 'mini_histogram'
module DerailedBenchmarks
# A class used to read several benchmark files
# it will parse each file, then sort by average
# time of benchmarks. It can be used to find
# the fastest and slowest examples and give information
# about them such as what the percent difference is
# and if the results are statistically significant
#
# Example:
#
# branch_info = {}
# branch_info["loser"] = { desc: "Old commit", time: Time.now, file: dir.join("loser.bench.txt"), name: "loser" }
# branch_info["winner"] = { desc: "I am the new commit", time: Time.now + 1, file: dir.join("winner.bench.txt"), name: "winner" }
# stats = DerailedBenchmarks::StatsFromDir.new(branch_info)
#
# stats.newest.average # => 10.5
# stats.oldest.average # => 11.0
# stats.significant? # => true
# stats.x_faster # => "1.0476"
class StatsFromDir
FORMAT = "%0.4f"
attr_reader :stats, :oldest, :newest
def initialize(hash)
@files = []
hash.each do |branch, info_hash|
file = info_hash.fetch(:file)
desc = info_hash.fetch(:desc)
time = info_hash.fetch(:time)
@files << StatsForFile.new(file: file, desc: desc, time: time, name: branch)
end
@files.sort_by! { |f| f.time }
@oldest = @files.first
@newest = @files.last
end
def call
@files.each(&:call)
stats_95 = statistical_test(confidence: 95)
# If default check is good, see if we also pass a more rigorous test
# if so, then use the more rigourous test
if stats_95[:alternative]
stats_99 = statistical_test(confidence: 99)
@stats = stats_99 if stats_99[:alternative]
end
@stats ||= stats_95
self
end
def statistical_test(series_1=oldest.values, series_2=newest.values, confidence: 95)
StatisticalTest::KSTest.two_samples(
group_one: series_1,
group_two: series_2,
alpha: (100 - confidence) / 100.0
)
end
def significant?
@stats[:alternative]
end
def d_max
@stats[:d_max].to_f
end
def d_critical
@stats[:d_critical].to_f
end
def x_faster
(oldest.median/newest.median).to_f
end
def faster?
newest.median < oldest.median
end
def percent_faster
(((oldest.median - newest.median) / oldest.median).to_f * 100)
end
def change_direction
if faster?
"FASTER 🚀🚀🚀"
else
"SLOWER 🐢🐢🐢"
end
end
def align
" " * (percent_faster.to_s.index(".") - x_faster.to_s.index("."))
end
def histogram(io = $stdout)
newest_histogram = MiniHistogram.new(newest.values)
oldest_histogram = MiniHistogram.new(oldest.values)
MiniHistogram.set_average_edges!(newest_histogram, oldest_histogram)
{newest => newest_histogram, oldest => oldest_histogram}.each do |report, histogram|
plot = UnicodePlot.histogram(
histogram,
title: "\n#{' ' * 18 }Histogram - [#{report.name}] #{report.desc.inspect}",
ylabel: "Time (s)",
xlabel: "# of runs in range"
)
plot.render(io)
io.puts
end
io.puts
end
def banner(io = $stdout)
io.puts
if significant?
io.puts "❤️ ❤️ ❤️ (Statistically Significant) ❤️ ❤️ ❤️"
else
io.puts "👎👎👎(NOT Statistically Significant) 👎👎👎"
end
io.puts
io.puts "[#{newest.name}] #{newest.desc.inspect} - (#{newest.median} seconds)"
io.puts " #{change_direction} by:"
io.puts " #{align}#{FORMAT % x_faster}x [older/newer]"
io.puts " #{FORMAT % percent_faster}\% [(older - newer) / older * 100]"
io.puts "[#{oldest.name}] #{oldest.desc.inspect} - (#{oldest.median} seconds)"
io.puts
io.puts "Iterations per sample: #{ENV["TEST_COUNT"]}"
io.puts "Samples: #{newest.values.length}"
io.puts
io.puts "Test type: Kolmogorov Smirnov"
io.puts "Confidence level: #{@stats[:confidence_level] * 100} %"
io.puts "Is significant? (max > critical): #{significant?}"
io.puts "D critical: #{d_critical}"
io.puts "D max: #{d_max}"
histogram(io)
io.puts
end
end
end
|
