1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
|
#!/usr/bin/python
"""Compare failed tests in CTS/VTS test_result.xml.
Given two test_result.xml's (A and B), this script lists all failed tests in A,
and shows result of the same test in B.
"""
import argparse
import collections
import csv
import xml.etree.ElementTree as ET
PASS = 'pass'
FAIL = 'fail'
NO_DATA = 'no_data'
ATTRS_TO_SHOW = ['Result::Build.build_model',
'Result::Build.build_id',
'Result.suite_name',
'Result.suite_plan',
'Result.suite_build_number',
'Result.start_display',
'Result::Build.build_abis_32',
'Result::Build.build_abis_64',]
def parse_attrib_path(attrib_path):
first_dot = attrib_path.index('.')
tags = attrib_path[:first_dot].split('::')
attr_name = attrib_path[first_dot+1:]
return tags, attr_name
def get_test_info(root):
"""Get test info from test_result.xml."""
test_info = collections.OrderedDict()
for attrib_path in ATTRS_TO_SHOW:
tags, attr_name = parse_attrib_path(attrib_path)
node = root
while True:
tags = tags[1:]
if tags:
node = node.find(tags[0])
else:
break
test_info[attr_name] = node.attrib[attr_name]
return test_info
def print_test_infos(test_result_a, test_result_b):
"""Print test infomation of both results in table format."""
info_a = test_result_a['info']
info_b = test_result_b['info']
max_key_len = max([len(k) for k in info_a])
max_value_a_len = max([len(info_a[k]) for k in info_a])
max_value_b_len = max([len(info_b[k]) for k in info_b])
table_len = (max_key_len + 2 + max_value_a_len + 2 + max_value_b_len)
line_format = '{:{}} {:{}} {}'
print '=' * table_len
for key in info_a:
print line_format.format(key, max_key_len,
info_a[key], max_value_a_len,
info_b[key])
print '=' * table_len
print
def get_result(test_result, module_name, testcase_name, test_name):
"""Get result of specifc module, testcase and test name."""
modules = test_result['modules']
if module_name not in modules:
return NO_DATA
testcases = modules[module_name]
if testcase_name not in testcases:
return NO_DATA
tests = testcases[testcase_name]
if test_name not in tests:
return NO_DATA
return ', '.join([x + ': ' + y for x, y in tests[test_name].items()])
def read_test_result_xml(test_result_path):
"""Given the path to a test_result.xml, read that into a ordered dict."""
tree = ET.parse(test_result_path)
root = tree.getroot()
test_result = collections.OrderedDict()
test_result['info'] = get_test_info(root)
modules = collections.OrderedDict()
test_result['modules'] = modules
for module in root.iter('Module'):
abi = module.attrib['abi']
module_name = module.attrib['name']
if not module_name in modules:
modules[module_name] = collections.OrderedDict()
testcases = modules[module_name]
for testcase in module.iter('TestCase'):
testcase_name = testcase.attrib['name']
if not testcase_name in testcases:
testcases[testcase_name] = collections.OrderedDict()
tests = testcases[testcase_name]
for test in testcase.iter('Test'):
test_name = test.attrib['name']
if not test_name in tests:
tests[test_name] = collections.OrderedDict()
if abi in tests[test_name]:
print '[WARNING] duplicated test:', test_name
tests[test_name][abi] = test.attrib['result']
return test_result
def compare_failed_tests(test_result_a, test_result_b, csvfile):
"""Do the comparison.
Given two test result dicts (A and B), list all failed test in A and display
result of the same test in B.
Args:
test_result_a: the dict returned from read_test_result(test_result_a.xml)
test_result_b: the dict returned from read_test_result(test_result_b.xml)
csvfile: a opened file
Returns:
string: diff report, summary
"""
writer = csv.writer(csvfile)
writer.writerow(['module', 'testcase', 'test', 'result in B'])
summary = ''
modules = test_result_a['modules']
for module_name, testcases in modules.iteritems():
module_sub_summary = ''
for testcase_name, tests in testcases.iteritems():
testcase_sub_summary = ''
for test_name, result in tests.iteritems():
if FAIL in result.values():
result_b = get_result(
test_result_b, module_name, testcase_name, test_name)
testcase_sub_summary += ' ' + test_name + ': ' + result_b + '\n'
writer.writerow([module_name, testcase_name, test_name, result_b])
if testcase_sub_summary:
module_sub_summary = ' ' + testcase_name + '\n' + testcase_sub_summary
if module_sub_summary:
summary += module_name + '\n' + module_sub_summary + '\n'
return summary
def main():
parser = argparse.ArgumentParser()
parser.add_argument('test_result_a', help='path to first test_result.xml')
parser.add_argument('test_result_b', help='path to second test_result.xml')
parser.add_argument('--csv', default='diff.csv', help='path to csv output')
args = parser.parse_args()
test_result_a = read_test_result_xml(args.test_result_a)
test_result_b = read_test_result_xml(args.test_result_b)
print_test_infos(test_result_a, test_result_b)
with open(args.csv, 'w') as csvfile:
summary = compare_failed_tests(test_result_a, test_result_b, csvfile)
print summary
if __name__ == '__main__':
main()
|
