""" Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. See https://llvm.org/LICENSE.txt for license information. SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception Provides the LLDBTestResult class, which holds information about progress and results of a single test run. """ from __future__ import absolute_import from __future__ import print_function # System modules import inspect import os # Third-party modules import unittest2 # LLDB Modules from . import configuration from lldbsuite.test_event.event_builder import EventBuilder from lldbsuite.test_event import build_exception class LLDBTestResult(unittest2.TextTestResult): """ Enforce a singleton pattern to allow introspection of test progress. Overwrite addError(), addFailure(), and addExpectedFailure() methods to enable each test instance to track its failure/error status. It is used in the LLDB test framework to emit detailed trace messages to a log file for easier human inspection of test failures/errors. """ __singleton__ = None __ignore_singleton__ = False @staticmethod def getTerminalSize(): import os env = os.environ def ioctl_GWINSZ(fd): try: import fcntl import termios import struct import os cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) except: return return cr cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) if not cr: try: fd = os.open(os.ctermid(), os.O_RDONLY) cr = ioctl_GWINSZ(fd) os.close(fd) except: pass if not cr: cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) return int(cr[1]), int(cr[0]) def __init__(self, *args): if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: raise Exception("LLDBTestResult instantiated more than once") super(LLDBTestResult, self).__init__(*args) LLDBTestResult.__singleton__ = self # Now put this singleton into the lldb module namespace. configuration.test_result = self # Computes the format string for displaying the counter. counterWidth = len(str(configuration.suite.countTestCases())) self.fmt = "%" + str(counterWidth) + "d: " self.indentation = ' ' * (counterWidth + 2) # This counts from 1 .. suite.countTestCases(). self.counter = 0 (width, height) = LLDBTestResult.getTerminalSize() self.results_formatter = configuration.results_formatter_object def _config_string(self, test): compiler = getattr(test, "getCompiler", None) arch = getattr(test, "getArchitecture", None) return "%s-%s" % (compiler() if compiler else "", arch() if arch else "") def _exc_info_to_string(self, err, test): """Overrides superclass TestResult's method in order to append our test config info string to the exception info string.""" if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): return '%sConfig=%s-%s' % (super(LLDBTestResult, self)._exc_info_to_string(err, test), test.getArchitecture(), test.getCompiler()) else: return super(LLDBTestResult, self)._exc_info_to_string(err, test) def getDescription(self, test): doc_first_line = test.shortDescription() if self.descriptions and doc_first_line: return '\n'.join((str(test), self.indentation + doc_first_line)) else: return str(test) @staticmethod def _getFileBasedCategories(test): """ Returns the list of categories to which this test case belongs by looking for a ".categories" file. We start at the folder the test is in an traverse the hierarchy upwards - we guarantee a .categories to exist at the top level directory so we do not end up looping endlessly. """ import inspect import os.path folder = inspect.getfile(test.__class__) folder = os.path.dirname(folder) while folder != '/': categories_file_name = os.path.join(folder, ".categories") if os.path.exists(categories_file_name): categories_file = open(categories_file_name, 'r') categories = categories_file.readline() categories_file.close() categories = str.replace(categories, '\n', '') categories = str.replace(categories, '\r', '') return categories.split(',') else: folder = os.path.dirname(folder) continue def getCategoriesForTest(self, test): """ Gets all the categories for the currently running test method in test case """ test_categories = [] test_method = getattr(test, test._testMethodName) if test_method is not None and hasattr(test_method, "categories"): test_categories.extend(test_method.categories) test_categories.extend(self._getFileBasedCategories(test)) return test_categories def hardMarkAsSkipped(self, test): getattr(test, test._testMethodName).__func__.__unittest_skip__ = True getattr( test, test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" def checkExclusion(self, exclusion_list, name): if exclusion_list: import re for item in exclusion_list: if re.search(item, name): return True return False def startTest(self, test): if configuration.shouldSkipBecauseOfCategories( self.getCategoriesForTest(test)): self.hardMarkAsSkipped(test) if self.checkExclusion( configuration.skip_tests, test.id()): self.hardMarkAsSkipped(test) self.counter += 1 test.test_number = self.counter if self.showAll: self.stream.write(self.fmt % self.counter) super(LLDBTestResult, self).startTest(test) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_start(test)) def addSuccess(self, test): if self.checkExclusion( configuration.xfail_tests, test.id()): self.addUnexpectedSuccess(test, None) return super(LLDBTestResult, self).addSuccess(test) if configuration.parsable: self.stream.write( "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_success(test)) def _isBuildError(self, err_tuple): exception = err_tuple[1] return isinstance(exception, build_exception.BuildError) def _getTestPath(self, test): if test is None: return "" elif hasattr(test, "test_filename"): return test.test_filename else: return inspect.getsourcefile(test.__class__) def _saveBuildErrorTuple(self, test, err): # Adjust the error description so it prints the build command and build error # rather than an uninformative Python backtrace. build_error = err[1] error_description = "{}\nTest Directory:\n{}".format( str(build_error), os.path.dirname(self._getTestPath(test))) self.errors.append((test, error_description)) self._mirrorOutput = True def addError(self, test, err): configuration.sdir_has_content = True if self._isBuildError(err): self._saveBuildErrorTuple(test, err) else: super(LLDBTestResult, self).addError(test, err) method = getattr(test, "markError", None) if method: method() if configuration.parsable: self.stream.write( "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if self.results_formatter: # Handle build errors as a separate event type if self._isBuildError(err): error_event = EventBuilder.event_for_build_error(test, err) else: error_event = EventBuilder.event_for_error(test, err) self.results_formatter.handle_event(error_event) def addCleanupError(self, test, err): configuration.sdir_has_content = True super(LLDBTestResult, self).addCleanupError(test, err) method = getattr(test, "markCleanupError", None) if method: method() if configuration.parsable: self.stream.write( "CLEANUP ERROR: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_cleanup_error( test, err)) def addFailure(self, test, err): if self.checkExclusion( configuration.xfail_tests, test.id()): self.addExpectedFailure(test, err, None) return configuration.sdir_has_content = True super(LLDBTestResult, self).addFailure(test, err) method = getattr(test, "markFailure", None) if method: method() if configuration.parsable: self.stream.write( "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if configuration.useCategories: test_categories = self.getCategoriesForTest(test) for category in test_categories: if category in configuration.failuresPerCategory: configuration.failuresPerCategory[ category] = configuration.failuresPerCategory[category] + 1 else: configuration.failuresPerCategory[category] = 1 if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_failure(test, err)) def addExpectedFailure(self, test, err, bugnumber): configuration.sdir_has_content = True super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) method = getattr(test, "markExpectedFailure", None) if method: method(err, bugnumber) if configuration.parsable: self.stream.write( "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_expected_failure( test, err, bugnumber)) def addSkip(self, test, reason): configuration.sdir_has_content = True super(LLDBTestResult, self).addSkip(test, reason) method = getattr(test, "markSkippedTest", None) if method: method() if configuration.parsable: self.stream.write( "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % (self._config_string(test), str(test), reason)) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_skip(test, reason)) def addUnexpectedSuccess(self, test, bugnumber): configuration.sdir_has_content = True super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) method = getattr(test, "markUnexpectedSuccess", None) if method: method(bugnumber) if configuration.parsable: self.stream.write( "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test))) if self.results_formatter: self.results_formatter.handle_event( EventBuilder.event_for_unexpected_success( test, bugnumber))