1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258
|
# Copyright (c) 2013 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Module containing base test results classes."""
import threading
class ResultType(object):
"""Class enumerating test types."""
# The test passed.
PASS = 'PASS'
# The test was intentionally skipped.
SKIP = 'SKIP'
# The test failed.
FAIL = 'FAIL'
# The test caused the containing process to crash.
CRASH = 'CRASH'
# The test timed out.
TIMEOUT = 'TIMEOUT'
# The test ran, but we couldn't determine what happened.
UNKNOWN = 'UNKNOWN'
# The test did not run.
NOTRUN = 'NOTRUN'
@staticmethod
def GetTypes():
"""Get a list of all test types."""
return [ResultType.PASS, ResultType.SKIP, ResultType.FAIL,
ResultType.CRASH, ResultType.TIMEOUT, ResultType.UNKNOWN,
ResultType.NOTRUN]
class BaseTestResult(object):
"""Base class for a single test result."""
def __init__(self, name, test_type, duration=0, log=''):
"""Construct a BaseTestResult.
Args:
name: Name of the test which defines uniqueness.
test_type: Type of the test result as defined in ResultType.
duration: Time it took for the test to run in milliseconds.
log: An optional string listing any errors.
"""
assert name
assert test_type in ResultType.GetTypes()
self._name = name
self._test_type = test_type
self._duration = duration
self._log = log
self._tombstones_url = None
self._logcat_url = None
def __str__(self):
return self._name
def __repr__(self):
return self._name
def __cmp__(self, other):
# pylint: disable=W0212
return cmp(self._name, other._name)
def __hash__(self):
return hash(self._name)
def SetName(self, name):
"""Set the test name.
Because we're putting this into a set, this should only be used if moving
this test result into another set.
"""
self._name = name
def GetName(self):
"""Get the test name."""
return self._name
def SetType(self, test_type):
"""Set the test result type."""
assert test_type in ResultType.GetTypes()
self._test_type = test_type
def GetType(self):
"""Get the test result type."""
return self._test_type
def GetDuration(self):
"""Get the test duration."""
return self._duration
def SetLog(self, log):
"""Set the test log."""
self._log = log
def GetLog(self):
"""Get the test log."""
return self._log
def SetTombstonesUrl(self, tombstones_url):
self._tombstones_url = tombstones_url
def GetTombstonesUrl(self):
return self._tombstones_url
def SetLogcatUrl(self, logcat_url):
self._logcat_url = logcat_url
def GetLogcatUrl(self):
return self._logcat_url
class TestRunResults(object):
"""Set of results for a test run."""
def __init__(self):
self._results = set()
self._results_lock = threading.RLock()
def GetLogs(self):
"""Get the string representation of all test logs."""
with self._results_lock:
s = []
for test_type in ResultType.GetTypes():
if test_type != ResultType.PASS:
for t in sorted(self._GetType(test_type)):
log = t.GetLog()
if log:
s.append('[%s] %s:' % (test_type, t))
s.append(log)
return '\n'.join(s)
def GetGtestForm(self):
"""Get the gtest string representation of this object."""
with self._results_lock:
s = []
plural = lambda n, s, p: '%d %s' % (n, p if n != 1 else s)
tests = lambda n: plural(n, 'test', 'tests')
s.append('[==========] %s ran.' % (tests(len(self.GetAll()))))
s.append('[ PASSED ] %s.' % (tests(len(self.GetPass()))))
skipped = self.GetSkip()
if skipped:
s.append('[ SKIPPED ] Skipped %s, listed below:' % tests(len(skipped)))
for t in sorted(skipped):
s.append('[ SKIPPED ] %s' % str(t))
all_failures = self.GetFail().union(self.GetCrash(), self.GetTimeout(),
self.GetUnknown())
if all_failures:
s.append('[ FAILED ] %s, listed below:' % tests(len(all_failures)))
for t in sorted(self.GetFail()):
s.append('[ FAILED ] %s' % str(t))
for t in sorted(self.GetCrash()):
s.append('[ FAILED ] %s (CRASHED)' % str(t))
for t in sorted(self.GetTimeout()):
s.append('[ FAILED ] %s (TIMEOUT)' % str(t))
for t in sorted(self.GetUnknown()):
s.append('[ FAILED ] %s (UNKNOWN)' % str(t))
s.append('')
s.append(plural(len(all_failures), 'FAILED TEST', 'FAILED TESTS'))
return '\n'.join(s)
def GetShortForm(self):
"""Get the short string representation of this object."""
with self._results_lock:
s = []
s.append('ALL: %d' % len(self._results))
for test_type in ResultType.GetTypes():
s.append('%s: %d' % (test_type, len(self._GetType(test_type))))
return ''.join([x.ljust(15) for x in s])
def __str__(self):
return self.GetGtestForm()
def AddResult(self, result):
"""Add |result| to the set.
Args:
result: An instance of BaseTestResult.
"""
assert isinstance(result, BaseTestResult)
with self._results_lock:
self._results.discard(result)
self._results.add(result)
def AddResults(self, results):
"""Add |results| to the set.
Args:
results: An iterable of BaseTestResult objects.
"""
with self._results_lock:
for t in results:
self.AddResult(t)
def AddTestRunResults(self, results):
"""Add the set of test results from |results|.
Args:
results: An instance of TestRunResults.
"""
assert isinstance(results, TestRunResults), (
'Expected TestRunResult object: %s' % type(results))
with self._results_lock:
# pylint: disable=W0212
self._results.update(results._results)
def GetAll(self):
"""Get the set of all test results."""
with self._results_lock:
return self._results.copy()
def _GetType(self, test_type):
"""Get the set of test results with the given test type."""
with self._results_lock:
return set(t for t in self._results if t.GetType() == test_type)
def GetPass(self):
"""Get the set of all passed test results."""
return self._GetType(ResultType.PASS)
def GetSkip(self):
"""Get the set of all skipped test results."""
return self._GetType(ResultType.SKIP)
def GetFail(self):
"""Get the set of all failed test results."""
return self._GetType(ResultType.FAIL)
def GetCrash(self):
"""Get the set of all crashed test results."""
return self._GetType(ResultType.CRASH)
def GetTimeout(self):
"""Get the set of all timed out test results."""
return self._GetType(ResultType.TIMEOUT)
def GetUnknown(self):
"""Get the set of all unknown test results."""
return self._GetType(ResultType.UNKNOWN)
def GetNotPass(self):
"""Get the set of all non-passed test results."""
return self.GetAll() - self.GetPass()
def DidRunPass(self):
"""Return whether the test run was successful."""
return not self.GetNotPass() - self.GetSkip()
|