1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
|
import argparse
from collections import defaultdict
from utils import from_markdown_table, to_markdown_table
def main():
parser = argparse.ArgumentParser(
"Main script to compare results from the benchmarks"
)
parser.add_argument(
"--before",
type=str,
default="before.txt",
help="Text file containing the times to use as base",
)
parser.add_argument(
"--after",
type=str,
default="after.txt",
help="Text file containing the times to use as new version",
)
parser.add_argument(
"--output", type=str, default="", help="Text file where to write the output"
)
args = parser.parse_args()
with open(args.before) as f:
content = f.read()
res_before = from_markdown_table(content)
with open(args.after) as f:
content = f.read()
res_after = from_markdown_table(content)
diff = defaultdict(defaultdict)
for model in res_before:
for task in res_before[model]:
mean_before, var_before = res_before[model][task]
if task not in res_after[model]:
diff[model][task] = (None, mean_before, var_before, None, None)
else:
mean_after, var_after = res_after[model][task]
diff[model][task] = (
mean_before / mean_after,
mean_before,
var_before,
mean_after,
var_after,
)
for model in res_after:
for task in res_after[model]:
if task not in res_before[model]:
mean_after, var_after = res_after[model][task]
diff[model][task] = (None, None, None, mean_after, var_after)
header = (
"model",
"task",
"speedup",
"mean (before)",
"var (before)",
"mean (after)",
"var (after)",
)
out = to_markdown_table(diff, header=header)
print(out)
if args.output:
with open(args.output, "w") as f:
f.write(out)
if __name__ == "__main__":
main()
|
