1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351
|
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
DEFAULT_TIMEOUT = 10 # seconds
LONG_TIMEOUT = 60 # seconds
import os
import mozinfo
from wptmanifest.parser import atoms
atom_reset = atoms["Reset"]
enabled_tests = set(["testharness", "reftest", "wdspec"])
class Result(object):
def __init__(self, status, message, expected=None, extra=None):
if status not in self.statuses:
raise ValueError("Unrecognised status %s" % status)
self.status = status
self.message = message
self.expected = expected
self.extra = extra
def __repr__(self):
return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.status)
class SubtestResult(object):
def __init__(self, name, status, message, stack=None, expected=None):
self.name = name
if status not in self.statuses:
raise ValueError("Unrecognised status %s" % status)
self.status = status
self.message = message
self.stack = stack
self.expected = expected
def __repr__(self):
return "<%s.%s %s %s>" % (self.__module__, self.__class__.__name__, self.name, self.status)
class TestharnessResult(Result):
default_expected = "OK"
statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class TestharnessSubtestResult(SubtestResult):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "TIMEOUT", "NOTRUN"])
class ReftestResult(Result):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class WdspecResult(Result):
default_expected = "OK"
statuses = set(["OK", "ERROR", "TIMEOUT", "EXTERNAL-TIMEOUT", "CRASH"])
class WdspecSubtestResult(SubtestResult):
default_expected = "PASS"
statuses = set(["PASS", "FAIL", "ERROR"])
def get_run_info(metadata_root, product, **kwargs):
if product == "b2g":
return B2GRunInfo(metadata_root, product, **kwargs)
else:
return RunInfo(metadata_root, product, **kwargs)
class RunInfo(dict):
def __init__(self, metadata_root, product, debug, extras=None):
self._update_mozinfo(metadata_root)
self.update(mozinfo.info)
self["product"] = product
if debug is not None:
self["debug"] = debug
elif "debug" not in self:
# Default to release
self["debug"] = False
if extras is not None:
self.update(extras)
def _update_mozinfo(self, metadata_root):
"""Add extra build information from a mozinfo.json file in a parent
directory"""
path = metadata_root
dirs = set()
while path != os.path.expanduser('~'):
if path in dirs:
break
dirs.add(str(path))
path = os.path.split(path)[0]
mozinfo.find_and_update_from_json(*dirs)
class B2GRunInfo(RunInfo):
def __init__(self, *args, **kwargs):
RunInfo.__init__(self, *args, **kwargs)
self["os"] = "b2g"
class Test(object):
result_cls = None
subtest_result_cls = None
test_type = None
def __init__(self, tests_root, url, inherit_metadata, test_metadata,
timeout=DEFAULT_TIMEOUT, path=None, protocol="http"):
self.tests_root = tests_root
self.url = url
self._inherit_metadata = inherit_metadata
self._test_metadata = test_metadata
self.timeout = timeout
self.path = path
self.environment = {"protocol": protocol, "prefs": self.prefs}
def __eq__(self, other):
return self.id == other.id
@classmethod
def from_manifest(cls, manifest_item, inherit_metadata, test_metadata):
timeout = LONG_TIMEOUT if manifest_item.timeout == "long" else DEFAULT_TIMEOUT
protocol = "https" if hasattr(manifest_item, "https") and manifest_item.https else "http"
return cls(manifest_item.source_file.tests_root,
manifest_item.url,
inherit_metadata,
test_metadata,
timeout=timeout,
path=manifest_item.path,
protocol=protocol)
@property
def id(self):
return self.url
@property
def keys(self):
return tuple()
@property
def abs_path(self):
return os.path.join(self.tests_root, self.path)
def _get_metadata(self, subtest=None):
if self._test_metadata is not None and subtest is not None:
return self._test_metadata.get_subtest(subtest)
else:
return self._test_metadata
def itermeta(self, subtest=None):
for metadata in self._inherit_metadata:
yield metadata
if self._test_metadata is not None:
yield self._get_metadata()
if subtest is not None:
subtest_meta = self._get_metadata(subtest)
if subtest_meta is not None:
yield subtest_meta
def disabled(self, subtest=None):
for meta in self.itermeta(subtest):
disabled = meta.disabled
if disabled is not None:
return disabled
return None
@property
def restart_after(self):
for meta in self.itermeta(None):
restart_after = meta.restart_after
if restart_after is not None:
return True
return False
@property
def tags(self):
tags = set()
for meta in self.itermeta():
meta_tags = meta.tags
if atom_reset in meta_tags:
tags = meta_tags.copy()
tags.remove(atom_reset)
else:
tags |= meta_tags
tags.add("dir:%s" % self.id.lstrip("/").split("/")[0])
return tags
@property
def prefs(self):
prefs = {}
for meta in self.itermeta():
meta_prefs = meta.prefs
if atom_reset in prefs:
prefs = meta_prefs.copy()
del prefs[atom_reset]
else:
prefs.update(meta_prefs)
return prefs
def expected(self, subtest=None):
if subtest is None:
default = self.result_cls.default_expected
else:
default = self.subtest_result_cls.default_expected
metadata = self._get_metadata(subtest)
if metadata is None:
return default
try:
return metadata.get("expected")
except KeyError:
return default
def __repr__(self):
return "<%s.%s %s>" % (self.__module__, self.__class__.__name__, self.id)
class TestharnessTest(Test):
result_cls = TestharnessResult
subtest_result_cls = TestharnessSubtestResult
test_type = "testharness"
@property
def id(self):
return self.url
class ManualTest(Test):
test_type = "manual"
@property
def id(self):
return self.url
class ReftestTest(Test):
result_cls = ReftestResult
test_type = "reftest"
def __init__(self, tests_root, url, inherit_metadata, test_metadata, references,
timeout=DEFAULT_TIMEOUT, path=None, viewport_size=None,
dpi=None, protocol="http"):
Test.__init__(self, tests_root, url, inherit_metadata, test_metadata, timeout,
path, protocol)
for _, ref_type in references:
if ref_type not in ("==", "!="):
raise ValueError
self.references = references
self.viewport_size = viewport_size
self.dpi = dpi
@classmethod
def from_manifest(cls,
manifest_test,
inherit_metadata,
test_metadata,
nodes=None,
references_seen=None):
timeout = LONG_TIMEOUT if manifest_test.timeout == "long" else DEFAULT_TIMEOUT
if nodes is None:
nodes = {}
if references_seen is None:
references_seen = set()
url = manifest_test.url
node = cls(manifest_test.source_file.tests_root,
manifest_test.url,
inherit_metadata,
test_metadata,
[],
timeout=timeout,
path=manifest_test.path,
viewport_size=manifest_test.viewport_size,
dpi=manifest_test.dpi,
protocol="https" if hasattr(manifest_test, "https") and manifest_test.https else "http")
nodes[url] = node
for ref_url, ref_type in manifest_test.references:
comparison_key = (ref_type,) + tuple(sorted([url, ref_url]))
if ref_url in nodes:
manifest_node = ref_url
if comparison_key in references_seen:
# We have reached a cycle so stop here
# Note that just seeing a node for the second time is not
# enough to detect a cycle because
# A != B != C != A must include C != A
# but A == B == A should not include the redundant B == A.
continue
references_seen.add(comparison_key)
manifest_node = manifest_test.manifest.get_reference(ref_url)
if manifest_node:
reference = ReftestTest.from_manifest(manifest_node,
[],
None,
nodes,
references_seen)
else:
reference = ReftestTest(manifest_test.source_file.tests_root,
ref_url,
[],
None,
[])
node.references.append((reference, ref_type))
return node
@property
def id(self):
return self.url
@property
def keys(self):
return ("reftype", "refurl")
class WdspecTest(Test):
result_cls = WdspecResult
subtest_result_cls = WdspecSubtestResult
test_type = "wdspec"
manifest_test_cls = {"reftest": ReftestTest,
"testharness": TestharnessTest,
"manual": ManualTest,
"wdspec": WdspecTest}
def from_manifest(manifest_test, inherit_metadata, test_metadata):
test_cls = manifest_test_cls[manifest_test.item_type]
return test_cls.from_manifest(manifest_test, inherit_metadata, test_metadata)
|