1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
|
def benchmark_template():
import ubelt as ub
import pandas as pd
import inspect
import timerit
import time
from fractions import Fraction
_perf_counter_ns = time.perf_counter_ns
# Some bookkeeping needs to be done to build a dictionary that maps the
# method names to the functions themselves.
method_lut = {}
def register_method(func):
method_lut[func.__name__] = func
return func
@register_method
def method_ns_frac1(n):
from fractions import Fraction
for _ in range(n):
Fraction(time.perf_counter_ns(), 1_000_000_000)
@register_method
def method_ns_frac2(n):
for _ in range(n):
Fraction(time.perf_counter_ns(), 1_000_000_000)
@register_method
def method_ns_frac3(n):
for _ in range(n):
Fraction(_perf_counter_ns(), 1_000_000_000)
@register_method
def method_ns_float(n):
for _ in range(n):
time.perf_counter_ns() / 1_000_000_000
@register_method
def method_perf_counter_raw(n):
for _ in range(n):
time.perf_counter()
@register_method
def method_perf_counter_ns_raw(n):
for _ in range(n):
time.perf_counter()
# Change params here to modify number of trials
ti = timerit.Timerit(100000, bestof=100, verbose=1)
# if True, record every trail run and show variance in seaborn
# if False, use the standard timerit min/mean measures
RECORD_ALL = True
# These are the parameters that we benchmark over
basis = {
'method': list(method_lut),
'n': [0, 16, 64, 128, 256, 1024],
# 'param_name': [param values],
}
xlabel = 'n'
# Set these to param labels that directly transfer to method kwargs
# kw_labels = ['n']
kw_labels = list(inspect.signature(ub.peek(method_lut.values())).parameters)
# Set these to empty lists if they are not used
group_labels = {
'style': [],
'size': [],
}
group_labels['hue'] = list(
(ub.oset(basis) - {xlabel}) - set.union(*map(set, group_labels.values())))
grid_iter = list(ub.named_product(basis))
# For each variation of your experiment, create a row.
rows = []
for params in grid_iter:
group_keys = {}
for gname, labels in group_labels.items():
group_keys[gname + '_key'] = ub.repr2(
ub.dict_isect(params, labels), compact=1, si=1)
key = ub.repr2(params, compact=1, si=1)
# Make any modifications you need to compute input kwargs for each
# method here.
kwargs = ub.dict_isect(params.copy(), kw_labels)
method = method_lut[params['method']]
# Timerit will run some user-specified number of loops.
# and compute time stats with similar methodology to timeit
for timer in ti.reset(key):
# Put any setup logic you dont want to time here.
# ...
with timer:
# Put the logic you want to time here
method(**kwargs)
if RECORD_ALL:
# Seaborn will show the variance if this is enabled, otherwise
# use the robust timerit mean / min times
# chunk_iter = ub.chunks(ti.times, ti.bestof)
# times = list(map(min, chunk_iter)) # TODO: timerit method for this
times = ti.robust_times()
for _time in times:
row = {
# 'mean': ti.mean(),
'time': _time,
'key': key,
**group_keys,
**params,
}
rows.append(row)
else:
row = {
'mean': ti.mean(),
'min': ti.min(),
'key': key,
**group_keys,
**params,
}
rows.append(row)
time_key = 'time' if RECORD_ALL else 'min'
# The rows define a long-form pandas data array.
# Data in long-form makes it very easy to use seaborn.
data = pd.DataFrame(rows)
data = data.sort_values(time_key)
if RECORD_ALL:
# Show the min / mean if we record all
min_times = data.groupby('key').min().rename({'time': 'min'}, axis=1)
mean_times = data.groupby('key')[['time']].mean().rename({'time': 'mean'}, axis=1)
stats_data = pd.concat([min_times, mean_times], axis=1)
stats_data = stats_data.sort_values('min')
else:
stats_data = data
USE_OPENSKILL = 0
if USE_OPENSKILL:
# Lets try a real ranking method
# https://github.com/OpenDebates/openskill.py
import openskill
method_ratings = {m: openskill.Rating() for m in basis['method']}
other_keys = sorted(set(stats_data.columns) - {'key', 'method', 'min', 'mean', 'hue_key', 'size_key', 'style_key'})
for params, variants in stats_data.groupby(other_keys):
variants = variants.sort_values('mean')
ranking = variants['method'].reset_index(drop=True)
mean_speedup = variants['mean'].max() / variants['mean']
stats_data.loc[mean_speedup.index, 'mean_speedup'] = mean_speedup
min_speedup = variants['min'].max() / variants['min']
stats_data.loc[min_speedup.index, 'min_speedup'] = min_speedup
if USE_OPENSKILL:
# The idea is that each setting of parameters is a game, and each
# "method" is a player. We rank the players by which is fastest,
# and update their ranking according to the Weng-Lin Bayes ranking
# model. This does not take the fact that some "games" (i.e.
# parameter settings) are more important than others, but it should
# be fairly robust on average.
old_ratings = [[r] for r in ub.take(method_ratings, ranking)]
new_values = openskill.rate(old_ratings) # Not inplace
new_ratings = [openskill.Rating(*new[0]) for new in new_values]
method_ratings.update(ub.dzip(ranking, new_ratings))
print('Statistics:')
print(stats_data)
if USE_OPENSKILL:
from openskill import predict_win
win_prob = predict_win([[r] for r in method_ratings.values()])
skill_agg = pd.Series(ub.dzip(method_ratings.keys(), win_prob)).sort_values(ascending=False)
print('Aggregated Rankings =\n{}'.format(skill_agg))
plot = True
if plot:
# import seaborn as sns
# kwplot autosns works well for IPython and script execution.
# not sure about notebooks.
import kwplot
sns = kwplot.autosns()
plt = kwplot.autoplt()
plotkw = {}
for gname, labels in group_labels.items():
if labels:
plotkw[gname] = gname + '_key'
# Your variables may change
ax = kwplot.figure(fnum=1, doclf=True).gca()
sns.lineplot(data=data, x=xlabel, y=time_key, marker='o', ax=ax, **plotkw)
ax.set_title('Benchmark Name')
ax.set_xlabel('Size (todo: A better x-variable description)')
ax.set_ylabel('Time (todo: A better y-variable description)')
# ax.set_xscale('log')
# ax.set_yscale('log')
try:
__IPYTHON__
except NameError:
plt.show()
if __name__ == '__main__':
"""
CommandLine:
python ~/code/timerit/examples/benchmark_template.py
"""
benchmark_template()
|
