1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346
|
import copy
import functools
import json
import os
import re
import sys
import xml.etree.ElementTree as ET
from collections import defaultdict
from types import MethodType
from typing import Any, List, Optional, TYPE_CHECKING, Union
import pytest
from _pytest.config import Config, filename_arg
from _pytest.config.argparsing import Parser
from _pytest.junitxml import _NodeReporter, bin_xml_escape, LogXML
from _pytest.python import Module
from _pytest.reports import TestReport
from _pytest.stash import StashKey
from _pytest.terminal import _get_raw_skip_reason
from pytest_shard_custom import pytest_addoptions as shard_addoptions, PytestShardPlugin
if TYPE_CHECKING:
from _pytest._code.code import ReprFileLocation
# a lot of this file is copied from _pytest.junitxml and modified to get rerun info
xml_key = StashKey["LogXMLReruns"]()
STEPCURRENT_CACHE_DIR = "cache/stepcurrent"
def pytest_addoption(parser: Parser) -> None:
group = parser.getgroup("general")
group.addoption("--scs", action="store", default=None, dest="stepcurrent_skip")
group.addoption("--sc", action="store", default=None, dest="stepcurrent")
group.addoption("--rs", action="store", default=None, dest="run_single")
parser.addoption("--use-main-module", action="store_true")
group = parser.getgroup("terminal reporting")
group.addoption(
"--junit-xml-reruns",
action="store",
dest="xmlpath_reruns",
metavar="path",
type=functools.partial(filename_arg, optname="--junit-xml-reruns"),
default=None,
help="create junit-xml style report file at given path.",
)
group.addoption(
"--junit-prefix-reruns",
action="store",
metavar="str",
default=None,
help="prepend prefix to classnames in junit-xml output",
)
parser.addini(
"junit_suite_name_reruns", "Test suite name for JUnit report", default="pytest"
)
parser.addini(
"junit_logging_reruns",
"Write captured log messages to JUnit report: "
"one of no|log|system-out|system-err|out-err|all",
default="no",
)
parser.addini(
"junit_log_passing_tests_reruns",
"Capture log information for passing tests to JUnit report: ",
type="bool",
default=True,
)
parser.addini(
"junit_duration_report_reruns",
"Duration time to report: one of total|call",
default="total",
)
parser.addini(
"junit_family_reruns",
"Emit XML for schema: one of legacy|xunit1|xunit2",
default="xunit2",
)
shard_addoptions(parser)
def pytest_configure(config: Config) -> None:
xmlpath = config.option.xmlpath_reruns
# Prevent opening xmllog on worker nodes (xdist).
if xmlpath and not hasattr(config, "workerinput"):
junit_family = config.getini("junit_family_reruns")
config.stash[xml_key] = LogXMLReruns(
xmlpath,
config.option.junitprefix,
config.getini("junit_suite_name_reruns"),
config.getini("junit_logging_reruns"),
config.getini("junit_duration_report_reruns"),
junit_family,
config.getini("junit_log_passing_tests_reruns"),
)
config.pluginmanager.register(config.stash[xml_key])
if config.getoption("stepcurrent_skip"):
config.option.stepcurrent = config.getoption("stepcurrent_skip")
if config.getoption("run_single"):
config.option.stepcurrent = config.getoption("run_single")
if config.getoption("stepcurrent"):
config.pluginmanager.register(StepcurrentPlugin(config), "stepcurrentplugin")
if config.getoption("num_shards"):
config.pluginmanager.register(PytestShardPlugin(config), "pytestshardplugin")
def pytest_unconfigure(config: Config) -> None:
xml = config.stash.get(xml_key, None)
if xml:
del config.stash[xml_key]
config.pluginmanager.unregister(xml)
class _NodeReporterReruns(_NodeReporter):
def _prepare_content(self, content: str, header: str) -> str:
return content
def _write_content(self, report: TestReport, content: str, jheader: str) -> None:
if content == "":
return
tag = ET.Element(jheader)
tag.text = bin_xml_escape(content)
self.append(tag)
def append_skipped(self, report: TestReport) -> None:
# Referenced from the below
# https://github.com/pytest-dev/pytest/blob/2178ee86d7c1ee93748cfb46540a6e40b4761f2d/src/_pytest/junitxml.py#L236C6-L236C6
# Modified to escape characters not supported by xml in the skip reason. Everything else should be the same.
if hasattr(report, "wasxfail"):
# Super here instead of the actual code so we can reduce possible divergence
super().append_skipped(report)
else:
assert isinstance(report.longrepr, tuple)
filename, lineno, skipreason = report.longrepr
if skipreason.startswith("Skipped: "):
skipreason = skipreason[9:]
details = f"{filename}:{lineno}: {skipreason}"
skipped = ET.Element(
"skipped", type="pytest.skip", message=bin_xml_escape(skipreason)
)
skipped.text = bin_xml_escape(details)
self.append(skipped)
self.write_captured_output(report)
class LogXMLReruns(LogXML):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def append_rerun(self, reporter: _NodeReporter, report: TestReport) -> None:
if hasattr(report, "wasxfail"):
reporter._add_simple("skipped", "xfail-marked test passes unexpectedly")
else:
assert report.longrepr is not None
reprcrash: Optional[ReprFileLocation] = getattr(
report.longrepr, "reprcrash", None
)
if reprcrash is not None:
message = reprcrash.message
else:
message = str(report.longrepr)
message = bin_xml_escape(message)
reporter._add_simple("rerun", message, str(report.longrepr))
def pytest_runtest_logreport(self, report: TestReport) -> None:
super().pytest_runtest_logreport(report)
if report.outcome == "rerun":
reporter = self._opentestcase(report)
self.append_rerun(reporter, report)
if report.outcome == "skipped":
if isinstance(report.longrepr, tuple):
fspath, lineno, reason = report.longrepr
reason = f"{report.nodeid}: {_get_raw_skip_reason(report)}"
report.longrepr = (fspath, lineno, reason)
def node_reporter(self, report: Union[TestReport, str]) -> _NodeReporterReruns:
nodeid: Union[str, TestReport] = getattr(report, "nodeid", report)
# Local hack to handle xdist report order.
workernode = getattr(report, "node", None)
key = nodeid, workernode
if key in self.node_reporters:
# TODO: breaks for --dist=each
return self.node_reporters[key]
reporter = _NodeReporterReruns(nodeid, self)
self.node_reporters[key] = reporter
self.node_reporters_ordered.append(reporter)
return reporter
# imitating summary_failures in pytest's terminal.py
# both hookwrapper and tryfirst to make sure this runs before pytest's
@pytest.hookimpl(hookwrapper=True, tryfirst=True)
def pytest_terminal_summary(terminalreporter, exitstatus, config):
# prints stack traces for reruns
if terminalreporter.config.option.tbstyle != "no":
reports = terminalreporter.getreports("rerun")
if reports:
terminalreporter.write_sep("=", "RERUNS")
if terminalreporter.config.option.tbstyle == "line":
for rep in reports:
line = terminalreporter._getcrashline(rep)
terminalreporter.write_line(line)
else:
for rep in reports:
msg = terminalreporter._getfailureheadline(rep)
terminalreporter.write_sep("_", msg, red=True, bold=True)
terminalreporter._outrep_summary(rep)
terminalreporter._handle_teardown_sections(rep.nodeid)
yield
@pytest.hookimpl(tryfirst=True)
def pytest_pycollect_makemodule(module_path, path, parent) -> Module:
if parent.config.getoption("--use-main-module"):
mod = Module.from_parent(parent, path=module_path)
mod._getobj = MethodType(lambda x: sys.modules["__main__"], mod)
return mod
@pytest.hookimpl(hookwrapper=True)
def pytest_report_teststatus(report, config):
# Add the test time to the verbose output, unforunately I don't think this
# includes setup or teardown
pluggy_result = yield
if not isinstance(report, pytest.TestReport):
return
outcome, letter, verbose = pluggy_result.get_result()
if verbose:
pluggy_result.force_result(
(outcome, letter, f"{verbose} [{report.duration:.4f}s]")
)
@pytest.hookimpl(trylast=True)
def pytest_collection_modifyitems(items: List[Any]) -> None:
"""
This hook is used when rerunning disabled tests to get rid of all skipped tests
instead of running and skipping them N times. This avoids flooding the console
and XML outputs with junk. So we want this to run last when collecting tests.
"""
rerun_disabled_tests = os.getenv("PYTORCH_TEST_RERUN_DISABLED_TESTS", "0") == "1"
if not rerun_disabled_tests:
return
disabled_regex = re.compile(r"(?P<test_name>.+)\s+\([^\.]+\.(?P<test_class>.+)\)")
disabled_tests = defaultdict(set)
# This environment has already been set by run_test before it calls pytest
disabled_tests_file = os.getenv("DISABLED_TESTS_FILE", "")
if not disabled_tests_file or not os.path.exists(disabled_tests_file):
return
with open(disabled_tests_file) as fp:
for disabled_test in json.load(fp):
m = disabled_regex.match(disabled_test)
if m:
test_name = m["test_name"]
test_class = m["test_class"]
disabled_tests[test_class].add(test_name)
# When rerunning disabled test, ignore all test cases that are not disabled
filtered_items = []
for item in items:
test_name = item.name
test_class = item.parent.name
if (
test_class not in disabled_tests
or test_name not in disabled_tests[test_class]
):
continue
cpy = copy.copy(item)
cpy._initrequest()
filtered_items.append(cpy)
items.clear()
# NB: Need to edit items directly here to have the list reflected back to pytest
items.extend(filtered_items)
class StepcurrentPlugin:
# Modified fromo _pytest/stepwise.py in order to save the currently running
# test instead of the last failed test
def __init__(self, config: Config) -> None:
self.config = config
self.report_status = ""
assert config.cache is not None
self.cache: pytest.Cache = config.cache
self.directory = f"{STEPCURRENT_CACHE_DIR}/{config.getoption('stepcurrent')}"
self.lastrun: Optional[str] = self.cache.get(self.directory, None)
self.initial_val = self.lastrun
self.skip: bool = config.getoption("stepcurrent_skip")
self.run_single: bool = config.getoption("run_single")
def pytest_collection_modifyitems(self, config: Config, items: List[Any]) -> None:
if not self.lastrun:
self.report_status = "Cannot find last run test, not skipping"
return
# check all item nodes until we find a match on last run
failed_index = None
for index, item in enumerate(items):
if item.nodeid == self.lastrun:
failed_index = index
if self.skip:
failed_index += 1
break
# If the previously failed test was not found among the test items,
# do not skip any tests.
if failed_index is None:
self.report_status = "previously run test not found, not skipping."
else:
self.report_status = f"skipping {failed_index} already run items."
deselected = items[:failed_index]
del items[:failed_index]
if self.run_single:
self.report_status += f" Running only {items[0].nodeid}"
deselected += items[1:]
del items[1:]
config.hook.pytest_deselected(items=deselected)
def pytest_report_collectionfinish(self) -> Optional[str]:
if self.config.getoption("verbose") >= 0 and self.report_status:
return f"stepcurrent: {self.report_status}"
return None
def pytest_runtest_protocol(self, item, nextitem) -> None:
self.lastrun = item.nodeid
self.cache.set(self.directory, self.lastrun)
def pytest_sessionfinish(self, session, exitstatus):
if exitstatus == 0 and not self.run_single:
self.cache.set(self.directory, self.initial_val)
|