1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
|
#!/usr/bin/python
import optparse
import sys
import sqlite3
import scipy.stats
import numpy
import adbutil
from devices import DEVICES
DB_PATH="/data/data/com.android.benchmark/databases/BenchmarkResults"
OUT_PATH = "db/"
QUERY_BAD_FRAME = ("select run_id, name, total_duration from ui_results "
"where total_duration >=12 order by run_id, name")
QUERY_PERCENT_JANK = ("select run_id, name, sum(jank_frame) as jank_count, count (*) as total "
"from ui_results group by run_id, name")
class IterationResult:
def __init__(self):
self.durations = []
self.jank_count = 0
self.total_count = 0
def get_scoremap(dbpath):
db = sqlite3.connect(dbpath)
rows = db.execute(QUERY_BAD_FRAME)
scoremap = {}
for row in rows:
run_id = row[0]
name = row[1]
total_duration = row[2]
if not run_id in scoremap:
scoremap[run_id] = {}
if not name in scoremap[run_id]:
scoremap[run_id][name] = IterationResult()
scoremap[run_id][name].durations.append(float(total_duration))
for row in db.execute(QUERY_PERCENT_JANK):
run_id = row[0]
name = row[1]
jank_count = row[2]
total_count = row[3]
if run_id in scoremap.keys() and name in scoremap[run_id].keys():
scoremap[run_id][name].jank_count = long(jank_count)
scoremap[run_id][name].total_count = long(total_count)
db.close()
return scoremap
def score_device(name, serial, pull = False, verbose = False):
dbpath = OUT_PATH + name + ".db"
if pull:
adbutil.root(serial)
adbutil.pull(serial, DB_PATH, dbpath)
scoremap = None
try:
scoremap = get_scoremap(dbpath)
except sqlite3.DatabaseError:
print "Database corrupt, fetching..."
adbutil.root(serial)
adbutil.pull(serial, DB_PATH, dbpath)
scoremap = get_scoremap(dbpath)
per_test_score = {}
per_test_sample_count = {}
global_overall = {}
for run_id in iter(scoremap):
overall = []
if len(scoremap[run_id]) < 1:
if verbose:
print "Skipping short run %s" % run_id
continue
print "Run: %s" % run_id
for test in iter(scoremap[run_id]):
if verbose:
print "\t%s" % test
scores = []
sample_count = 0
res = scoremap[run_id][test]
stddev = numpy.std(res.durations)
mean = numpy.mean(res.durations)
sample_count = len(res.durations)
pj = 100 * res.jank_count / float(res.total_count)
score = stddev * mean *pj
if score == 0:
score = 1
scores.append(score)
if verbose:
print "\tScore = %f x %f x %f = %f (%d samples)" % (stddev, mean, pj, score, len(res.durations))
geo_run = scipy.stats.gmean(scores)
if test not in per_test_score:
per_test_score[test] = []
if test not in per_test_sample_count:
per_test_sample_count[test] = []
per_test_score[test].append(geo_run)
per_test_sample_count[test].append(int(sample_count))
overall.append(geo_run)
if not verbose:
print "\t%s:\t%0.2f (%0.2f avg. sample count)" % (test, geo_run, sample_count)
else:
print "\tOverall:\t%0.2f (%0.2f avg. sample count)" % (geo_run, sample_count)
print ""
global_overall[run_id] = scipy.stats.gmean(overall)
print "Run Overall: %f" % global_overall[run_id]
print ""
print ""
print "Variability (CV) - %s:" % name
for test in per_test_score:
print "\t%s:\t%0.2f%% (%0.2f avg sample count)" % (test, 100 * scipy.stats.variation(per_test_score[test]), numpy.mean(per_test_sample_count[test]))
print "\tOverall: %0.2f%%" % (100 * scipy.stats.variation([x for x in global_overall.values()]))
print ""
def parse_options(argv):
usage = 'Usage: %prog [options]'
desc = 'Example: %prog'
parser = optparse.OptionParser(usage=usage, description=desc)
parser.add_option("-p", dest='pull', action="store_true")
parser.add_option("-d", dest='device', action="store")
parser.add_option("-v", dest='verbose', action="store_true")
options, categories = parser.parse_args(argv[1:])
return options
def main():
options = parse_options(sys.argv)
if options.device != None:
score_device(options.device, DEVICES[options.device], options.pull, options.verbose)
else:
for name, serial in DEVICES.iteritems():
print "======== %s =========" % name
score_device(name, serial, options.pull, options.verbose)
if __name__ == "__main__":
main()
|
