1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
|
import functools
import os
import warnings
try:
import lxml.etree
p = lxml.etree.XMLParser(huge_tree=True)
parse = functools.partial(lxml.etree.parse, parser=p)
except ImportError:
import xml.etree.ElementTree as ET
parse = ET.parse
warnings.warn(
"lxml was not found. `pip install lxml` to make this script run much faster"
)
def open_test_results(directory):
xmls = []
for root, _, files in os.walk(directory):
for file in files:
if file.endswith(".xml"):
tree = parse(f"{root}/{file}")
xmls.append(tree)
return xmls
def get_testcases(xmls):
testcases = []
for xml in xmls:
root = xml.getroot()
testcases.extend(list(root.iter("testcase")))
return testcases
def find(testcase, condition):
children = list(testcase.iter())
assert children[0] is testcase
children = children[1:]
return condition(children)
def skipped_test(testcase):
def condition(children):
return "skipped" in {child.tag for child in children}
return find(testcase, condition)
def passed_test(testcase):
def condition(children):
if len(children) == 0:
return True
tags = {child.tag for child in children}
return "skipped" not in tags and "failed" not in tags
return find(testcase, condition)
def key(testcase):
file = testcase.attrib.get("file", "UNKNOWN")
classname = testcase.attrib["classname"]
name = testcase.attrib["name"]
return "::".join([file, classname, name])
def get_passed_testcases(xmls):
testcases = get_testcases(xmls)
passed_testcases = [testcase for testcase in testcases if passed_test(testcase)]
return passed_testcases
def get_excluded_testcases(xmls):
testcases = get_testcases(xmls)
excluded_testcases = [t for t in testcases if excluded_testcase(t)]
return excluded_testcases
def excluded_testcase(testcase):
def condition(children):
for child in children:
if child.tag == "skipped":
if "Policy: we don't run" in child.attrib["message"]:
return True
return False
return find(testcase, condition)
def is_unexpected_success(testcase):
def condition(children):
for child in children:
if child.tag != "failure":
continue
is_unexpected_success = (
"unexpected success" in child.attrib["message"].lower()
)
if is_unexpected_success:
return True
return False
return find(testcase, condition)
MSG = "This test passed, maybe we can remove the skip from dynamo_test_failures.py"
def is_passing_skipped_test(testcase):
def condition(children):
for child in children:
if child.tag != "skipped":
continue
has_passing_skipped_test_msg = MSG in child.attrib["message"]
if has_passing_skipped_test_msg:
return True
return False
return find(testcase, condition)
# NB: not an unexpected success
def is_failure(testcase):
def condition(children):
for child in children:
if child.tag != "failure":
continue
is_unexpected_success = (
"unexpected success" in child.attrib["message"].lower()
)
if not is_unexpected_success:
return True
return False
return find(testcase, condition)
|
