1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
|
#!/usr/bin/env python
# /// script
# requires-python = ">=3.10"
# dependencies = [
# "matplotlib",
# "pyqt6",
# ]
# ///
"""This program shows parametrized `hyperfine` benchmark results as an
errorbar plot."""
import argparse
import json
import sys
import matplotlib.pyplot as plt
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("file", help="JSON file with benchmark results", nargs="+")
parser.add_argument(
"--parameter-name",
metavar="name",
type=str,
help="Deprecated; parameter names are now inferred from benchmark files",
)
parser.add_argument(
"--log-x", help="Use a logarithmic x (parameter) axis", action="store_true"
)
parser.add_argument(
"--log-time", help="Use a logarithmic time axis", action="store_true"
)
parser.add_argument(
"--titles", help="Comma-separated list of titles for the plot legend"
)
parser.add_argument("-o", "--output", help="Save image to the given filename.")
args = parser.parse_args()
if args.parameter_name is not None:
sys.stderr.write(
"warning: --parameter-name is deprecated; names are inferred from "
"benchmark results\n"
)
def die(msg):
sys.stderr.write(f"fatal: {msg}\n")
sys.exit(1)
def extract_parameters(results):
"""Return `(parameter_name: str, parameter_values: List[float])`."""
if not results:
die("no benchmark data to plot")
(names, values) = zip(*(unique_parameter(b) for b in results))
names = frozenset(names)
if len(names) != 1:
die(
f"benchmarks must all have the same parameter name, but found: {sorted(names)}"
)
return (next(iter(names)), list(values))
def unique_parameter(benchmark):
"""Return the unique parameter `(name: str, value: float)`, or die."""
params_dict = benchmark.get("parameters", {})
if not params_dict:
die("benchmarks must have exactly one parameter, but found none")
if len(params_dict) > 1:
die(
f"benchmarks must have exactly one parameter, but found multiple: {sorted(params_dict)}"
)
[(name, value)] = params_dict.items()
return (name, float(value))
parameter_name = None
for filename in args.file:
with open(filename) as f:
results = json.load(f)["results"]
(this_parameter_name, parameter_values) = extract_parameters(results)
if parameter_name is not None and this_parameter_name != parameter_name:
die(
f"files must all have the same parameter name, but found {parameter_name!r} vs. {this_parameter_name!r}"
)
parameter_name = this_parameter_name
times_mean = [b["mean"] for b in results]
times_stddev = [b["stddev"] for b in results]
plt.errorbar(x=parameter_values, y=times_mean, yerr=times_stddev, capsize=2)
plt.xlabel(parameter_name)
plt.ylabel("Time [s]")
if args.log_time:
plt.yscale("log")
else:
plt.ylim(0, None)
if args.log_x:
plt.xscale("log")
if args.titles:
plt.legend(args.titles.split(","))
if args.output:
plt.savefig(args.output)
else:
plt.show()
|
