1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
#!/usr/bin/env python3
"""
GNU Test Results Analyzer and Aggregator
This script analyzes and aggregates test results from the GNU test suite.
It parses JSON files containing test results (PASS/FAIL/SKIP/ERROR) and:
1. Counts the number of tests in each result category
2. Can aggregate results from multiple JSON files with priority ordering
3. Outputs shell export statements for use in GitHub Actions workflows
Priority order for aggregation (highest to lowest):
- PASS: Takes precedence over all other results (best outcome)
- FAIL: Takes precedence over ERROR and SKIP
- ERROR: Takes precedence over SKIP
- SKIP: Lowest priority
Usage:
- Single file:
python analyze-gnu-results.py test-results.json
- Multiple files (with aggregation):
python analyze-gnu-results.py file1.json file2.json
- With output file for aggregated results:
python analyze-gnu-results.py -o=output.json file1.json file2.json
Output:
Prints shell export statements for TOTAL, PASS, FAIL, SKIP, XPASS, and ERROR
that can be evaluated in a shell environment.
"""
import json
import sys
from collections import defaultdict
def get_priority(result):
"""Return a priority value for result status (lower is higher priority)"""
priorities = {
"PASS": 0, # PASS is highest priority (best result)
"FAIL": 1, # FAIL is second priority
"ERROR": 2, # ERROR is third priority
"SKIP": 3, # SKIP is lowest priority
}
return priorities.get(result, 4) # Unknown states have lowest priority
def aggregate_results(json_files):
"""
Aggregate test results from multiple JSON files.
Prioritizes results in the order: SKIP > ERROR > FAIL > PASS
"""
# Combined results dictionary
combined_results = defaultdict(dict)
# Process each JSON file
for json_file in json_files:
try:
with open(json_file, "r") as f:
data = json.load(f)
# For each utility and its tests
for utility, tests in data.items():
for test_name, result in tests.items():
# If this test hasn't been seen yet, add it
if test_name not in combined_results[utility]:
combined_results[utility][test_name] = result
else:
# If it has been seen, apply priority rules
current_priority = get_priority(
combined_results[utility][test_name]
)
new_priority = get_priority(result)
# Lower priority value means higher precedence
if new_priority < current_priority:
combined_results[utility][test_name] = result
except FileNotFoundError:
print(f"Warning: File '{json_file}' not found.", file=sys.stderr)
continue
except json.JSONDecodeError:
print(f"Warning: '{json_file}' is not a valid JSON file.", file=sys.stderr)
continue
return combined_results
def analyze_test_results(json_data):
"""
Analyze test results from GNU test suite JSON data.
Counts PASS, FAIL, SKIP results for all tests.
"""
# Counters for test results
total_tests = 0
pass_count = 0
fail_count = 0
skip_count = 0
xpass_count = 0 # Not in JSON data but included for compatibility
error_count = 0 # Not in JSON data but included for compatibility
# Analyze each utility's tests
for utility, tests in json_data.items():
for test_name, result in tests.items():
total_tests += 1
match result:
case "PASS":
pass_count += 1
case "FAIL":
fail_count += 1
case "SKIP":
skip_count += 1
case "ERROR":
error_count += 1
case "XPASS":
xpass_count += 1
# Return the statistics
return {
"TOTAL": total_tests,
"PASS": pass_count,
"FAIL": fail_count,
"SKIP": skip_count,
"XPASS": xpass_count,
"ERROR": error_count,
}
def main():
"""
Main function to process JSON files and export variables.
Supports both single file analysis and multi-file aggregation.
"""
# Check if file arguments were provided
if len(sys.argv) < 2:
print("Usage: python analyze-gnu-results.py <json> [json ...]")
print(" For multiple files, results will be aggregated")
print(" Priority SKIP > ERROR > FAIL > PASS")
sys.exit(1)
json_files = sys.argv[1:]
output_file = None
# Check if the first argument is an output file (starts with -)
if json_files[0].startswith("-o="):
output_file = json_files[0][3:]
json_files = json_files[1:]
# Process the files
if len(json_files) == 1:
# Single file analysis
try:
with open(json_files[0], "r") as file:
json_data = json.load(file)
results = analyze_test_results(json_data)
except FileNotFoundError:
print(f"Error: File '{json_files[0]}' not found.", file=sys.stderr)
sys.exit(1)
except json.JSONDecodeError:
print(
f"Error: '{json_files[0]}' is not a valid JSON file.", file=sys.stderr
)
sys.exit(1)
else:
# Multiple files - aggregate them
json_data = aggregate_results(json_files)
results = analyze_test_results(json_data)
# Save aggregated data if output file is specified
if output_file:
with open(output_file, "w") as f:
json.dump(json_data, f, indent=2)
# Print export statements for shell evaluation
print(f"export TOTAL={results['TOTAL']}")
print(f"export PASS={results['PASS']}")
print(f"export SKIP={results['SKIP']}")
print(f"export FAIL={results['FAIL']}")
print(f"export XPASS={results['XPASS']}")
print(f"export ERROR={results['ERROR']}")
if __name__ == "__main__":
main()
|
