#!/usr/bin/env python3 # Copyright 2017 the V8 project authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ Global system tests for V8 test runners and fuzzers. This hooks up the framework under tools/testrunner testing high-level scenarios with different test suite extensions and build configurations. """ # TODO(machenbach): Mock out util.GuessOS to make these tests really platform # independent. # TODO(machenbach): Move coverage recording to a global test entry point to # include other unittest suites in the coverage report. # TODO(machenbach): Coverage data from multiprocessing doesn't work. # TODO(majeski): Add some tests for the fuzzers. from collections import deque from pathlib import Path import re import sys import unittest from mock import patch TOOLS_ROOT = Path(__file__).resolve().parent.parent sys.path.append(str(TOOLS_ROOT)) from testrunner import standard_runner from testrunner import num_fuzzer from testrunner.testproc import base from testrunner.testproc import fuzzer from testrunner.testproc import resultdb from testrunner.testproc.resultdb_server_mock import RDBMockServer from testrunner.utils.test_utils import ( temp_base, TestRunnerTest, with_json_output, FakeOSContext, ) class StandardRunnerTest(TestRunnerTest): def setUp(self): self.mock_rdb_server = RDBMockServer() resultdb.TESTING_SINK = dict( auth_token='none', address=self.mock_rdb_server.address) def tearDown(self): resultdb.TESTING_SINK = None def get_runner_class(self): return standard_runner.StandardTestRunner def testPass(self): """Test running only passing tests in two variants. Also test printing durations. """ result = self.run_tests( '--progress=verbose', '--variants=default,stress', '--time', 'sweet/bananas', 'sweet/raspberries', ) result.stdout_includes('sweet/bananas default: PASS') # TODO(majeski): Implement for test processors # self.assertIn('Total time:', result.stderr, result) # self.assertIn('sweet/bananas', result.stderr, result) result.has_returncode(0) def testPassHeavy(self): """Test running with some tests marked heavy.""" result = self.run_tests( '--progress=verbose', '--variants=nooptimization', '-j2', 'sweet', baseroot='testroot3', ) result.stdout_includes('7 tests ran') result.has_returncode(0) def testShardedProc(self): for shard in [1, 2]: result = self.run_tests( '--progress=verbose', '--variants=default,stress', '--shard-count=2', '--shard-run=%d' % shard, 'sweet/blackberries', 'sweet/raspberries', infra_staging=False, ) # One of the shards gets one variant of each test. result.stdout_includes('2 tests ran') if shard == 1: result.stdout_includes('sweet/raspberries default') result.stdout_includes('sweet/raspberries stress') result.has_returncode(0) else: result.stdout_includes( 'sweet/blackberries default: FAIL') result.stdout_includes( 'sweet/blackberries stress: FAIL') result.has_returncode(1) @unittest.skip("incompatible with test processors") def testSharded(self): """Test running a particular shard.""" for shard in [1, 2]: result = self.run_tests( '--progress=verbose', '--variants=default,stress', '--shard-count=2', '--shard-run=%d' % shard, 'sweet/bananas', 'sweet/raspberries', ) # One of the shards gets one variant of each test. result.stdout_includes('Running 2 tests') result.stdout_includes('sweet/bananas') result.stdout_includes('sweet/raspberries') result.has_returncode(0) def testFail(self): """Test running only failing tests in two variants.""" result = self.run_tests( '--progress=verbose', '--variants=default,stress', 'sweet/strawberries', infra_staging=False, ) result.stdout_includes('sweet/strawberries default: FAIL') result.has_returncode(1) def testGN(self): """Test setup with legacy GN out dir.""" result = self.run_tests('--gn', baseroot="testroot5", outdir='out.gn') result.stdout_includes('>>> Latest GN build found: build') result.stdout_includes('Build found: ') result.stdout_includes('_v8_test/out.gn/build') result.has_returncode(2) def testMalformedJsonConfig(self): """Test running only failing tests in two variants.""" result = self.run_tests(baseroot="testroot4") result.stdout_includes('contains invalid json') result.stdout_includes('Failed to load build config') result.stderr_includes('testrunner.base_runner.TestRunnerError') result.has_returncode(5) def testFailWithRerunAndJSON(self): """Test re-running a failing test and output to json.""" result = self.run_tests( '--progress=verbose', '--variants=default', '--rerun-failures-count=2', '--random-seed=123', '--json-test-results', with_json_output, 'sweet/strawberries', infra_staging=False, ) result.stdout_includes('sweet/strawberries default: FAIL') # With test processors we don't count reruns as separated failures. # TODO(majeski): fix it? result.stdout_includes('1 tests failed') result.has_returncode(1) # TODO(majeski): Previously we only reported the variant flags in the # flags field of the test result. # After recent changes we report all flags, including the file names. # This is redundant to the command. Needs investigation. result.json_content_equals('expected_test_results1.json') def testRDB(self): with self.with_fake_rdb() as records: # sweet/bananaflakes fails first time on stress but passes on default def tag_dict(tags): return {t['key']: t['value'] for t in tags} self.run_tests( '--variants=default,stress', '--rerun-failures-count=2', '--time', 'sweet', baseroot='testroot2', infra_staging=False, ) self.assertEqual(len(records), 3) self.assertEqual(records[0]['testId'], 'sweet/bananaflakes//stress') self.assertEqual(tag_dict(records[0]['tags'])['run'], '1') self.assertFalse(records[0]['expected']) self.assertEqual(records[1]['testId'], 'sweet/bananaflakes//stress') self.assertEqual(tag_dict(records[1]['tags'])['run'], '2') self.assertTrue(records[1]['expected']) self.assertEqual(records[2]['testId'], 'sweet/bananaflakes//default') self.assertEqual(tag_dict(records[2]['tags'])['run'], '1') self.assertTrue(records[2]['expected']) def testFlakeWithRerunAndJSON(self): """Test re-running a failing test and output to json.""" result = self.run_tests( '--progress=verbose', '--variants=default', '--rerun-failures-count=2', '--random-seed=123', '--json-test-results', with_json_output, 'sweet', baseroot='testroot2', infra_staging=False, ) result.stdout_includes('sweet/bananaflakes default: FAIL PASS') result.stdout_includes('=== sweet/bananaflakes (flaky) ===') result.stdout_includes('1 tests failed') result.stdout_includes('1 tests were flaky') result.has_returncode(1) result.json_content_equals('expected_test_results2.json') self.assertTrue(re.search( r'sweet/bananaflakes default: FAIL \(\d+\.\d+:\d+\.\d+\)', result.test_schedule)) self.assertTrue(re.search( r'sweet/bananaflakes default: PASS \(\d+\.\d+:\d+\.\d+\)', result.test_schedule)) def testAutoDetect(self): """Fake a build with several auto-detected options. Using all those options at once doesn't really make much sense. This is merely for getting coverage. """ result = self.run_tests( '--progress=verbose', '--variants=default', 'sweet/bananas', config_overrides=dict( arch="ia32", asan=True, cfi=True, dcheck_always_on=True, has_webassembly=True, msan=True, target_cpu='x86', tsan=True, ubsan=True, use_sanitizer=True, v8_target_cpu='x86', )) result.stdout_includes('>>> Statusfile variables:') result.stdout_includes( "DEBUG_defined=False, arch=ia32, asan=True, byteorder=little, " "cfi=True, code_comments=False, component_build=False, " "dcheck_always_on=True, debug_code=False, debugging_features=False, " "deopt_fuzzer=False, device_type=None, " "dict_property_const_tracking=False, disassembler=False, " "endurance_fuzzer=False, full_debug=False, gc_fuzzer=False, " "gc_stress=False, gdbjit=False, has_jitless=False, has_maglev=False, " "has_turbofan=False, has_webassembly=True, i18n=True, " "interrupt_fuzzer=False, is_android=False, is_ios=False, " "isolates=False, lite_mode=False, mode=debug, msan=True, " "no_harness=False, no_simd_hardware=False, novfp3=False, " "optimize_for_size=False, simulator_run=False, slow_dchecks=False, " "system=linux, target_cpu=x86, tsan=True, ubsan=True, " "use_sanitizer=True, v8_target_cpu=x86, verify_heap=False, " "verify_predictable=False") result.stdout_includes('>>> Running tests for ia32.release') result.has_returncode(0) # TODO(machenbach): Test some more implications of the auto-detected # options, e.g. that the right env variables are set. def testLimitedPreloading(self): result = self.run_tests('--progress=verbose', '--variants=default', '-j1', 'sweet/*') result.stdout_includes("sweet/mangoes default: PASS") result.stdout_includes("sweet/cherries default: FAIL") result.stdout_includes("sweet/apples default: FAIL") result.stdout_includes("sweet/strawberries default: FAIL") result.stdout_includes("sweet/bananas default: PASS") result.stdout_includes("sweet/blackberries default: FAIL") result.stdout_includes("sweet/raspberries default: PASS") def testWithFakeContext(self): with patch( 'testrunner.local.context.find_os_context_factory', return_value=FakeOSContext): result = self.run_tests( '--progress=verbose', 'sweet', ) result.stdout_includes('===>Starting stuff') result.stdout_includes('>>> Running tests for x64.release\n' '>>> Running with test processors\n') result.stdout_includes('--- stdout ---\nfake stdout 1') result.stdout_includes('--- stderr ---\nfake stderr 1') result.stdout_includes('=== sweet/raspberries ===') result.stdout_includes('=== sweet/cherries ===') result.stdout_includes('=== sweet/apples ===') result.stdout_includes('Command: fake_wrapper ') result.stdout_includes( '===\n' '=== 4 tests failed\n' '===\n' '>>> 7 base tests produced 7 (100%) non-filtered tests\n' '>>> 7 tests ran\n' '<===Stopping stuff\n') def testSkips(self): """Test skipping tests in status file for a specific variant.""" result = self.run_tests( '--progress=verbose', '--variants=nooptimization', 'sweet/strawberries', infra_staging=False, ) result.stdout_includes('0 tests ran') result.has_returncode(2) def testRunSkips(self): """Inverse the above. Test parameter to keep running skipped tests.""" result = self.run_tests( '--progress=verbose', '--variants=nooptimization', '--run-skipped', 'sweet/strawberries', ) result.stdout_includes('1 tests failed') result.stdout_includes('1 tests ran') result.has_returncode(1) def testDefault(self): """Test using default test suites, though no tests are run since they don't exist in a test setting. """ result = self.run_tests( infra_staging=False, ) result.stdout_includes('0 tests ran') result.has_returncode(2) def testNoBuildConfig(self): """Test failing run when build config is not found.""" result = self.run_tests(baseroot='wrong_path', with_build_config=False) result.stdout_includes('Failed to load build config') result.has_returncode(5) def testInconsistentArch(self): """Test failing run when attempting to wrongly override the arch.""" result = self.run_tests('--arch=ia32') result.stdout_includes( '--arch value (ia32) inconsistent with build config (x64).') result.has_returncode(5) def testWrongVariant(self): """Test using a bogus variant.""" result = self.run_tests('--variants=meh') result.has_returncode(5) def testModeFromBuildConfig(self): """Test auto-detection of mode from build config.""" result = self.run_tests('--outdir=out/build', 'sweet/bananas') result.stdout_includes('Running tests for x64.release') result.has_returncode(0) def testPredictable(self): """Test running a test in verify-predictable mode. The test will fail because of missing allocation output. We verify that and that the predictable flags are passed and printed after failure. """ result = self.run_tests( '--progress=verbose', '--variants=default', 'sweet/bananas', infra_staging=False, config_overrides=dict(verify_predictable=True), ) result.stdout_includes('1 tests ran') result.stdout_includes('sweet/bananas default: FAIL') result.stdout_includes('Test had no allocation output') result.stdout_includes('--predictable --verify-predictable') result.has_returncode(1) def testSlowArch(self): """Test timeout factor manipulation on slow architecture.""" result = self.run_tests( '--progress=verbose', '--variants=default', 'sweet/bananas', config_overrides=dict(v8_target_cpu='arm64'), ) # TODO(machenbach): We don't have a way for testing if the correct # timeout was used. result.has_returncode(0) def testRandomSeedStressWithDefault(self): """Test using random-seed-stress feature has the right number of tests.""" result = self.run_tests( '--progress=verbose', '--variants=default', '--random-seed-stress-count=2', 'sweet/bananas', infra_staging=False, ) result.stdout_includes('2 tests ran') result.has_returncode(0) def testRandomSeedStressWithSeed(self): """Test using random-seed-stress feature passing a random seed.""" result = self.run_tests( '--progress=verbose', '--variants=default', '--random-seed-stress-count=2', '--random-seed=123', 'sweet/strawberries', ) result.stdout_includes('2 tests ran') # We use a failing test so that the command is printed and we can verify # that the right random seed was passed. result.stdout_includes('--random-seed=123') result.has_returncode(1) def testRandomSeedStressWithNumfuzz(self): """Test using random-seed-stress feature with numfuzz flavor as used by flake bisect for flakes on numfuzz. """ result = self.run_tests( '--progress=verbose', '--framework=num_fuzzer', '--variants=default', '--random-seed-stress-count=2', 'sweet/bananas', 'sweet/apples', infra_staging=False, baseroot='testroot7' ) # The bananas test is expected to pass when --fuzzing, one of the numfuzz # default flags is present. The apples test is expected to fail with this # flag. result.stdout_includes('sweet/bananas default: PASS') result.stdout_includes('sweet/apples default: FAIL') # We get everything twice due to the stress count above set to 2. result.stdout_includes('2 tests failed') result.stdout_includes('4 tests ran') result.has_returncode(1) def testSpecificVariants(self): """Test using NO_VARIANTS modifiers in status files skips the desire tests. The test runner cmd line configures 4 tests to run (2 tests * 2 variants). But the status file applies a modifier to each skipping one of the variants. """ result = self.run_tests( '--progress=verbose', '--variants=default,stress', 'sweet/bananas', 'sweet/raspberries', config_overrides=dict(asan=True), ) # Both tests are either marked as running in only default or only # slow variant. result.stdout_includes('2 tests ran') result.has_returncode(0) def testDotsProgress(self): result = self.run_tests( '--progress=dots', 'sweet/cherries', 'sweet/bananas', '--no-sorting', '-j1', # make results order deterministic infra_staging=False, ) result.stdout_includes('2 tests ran') result.stdout_includes('F.') result.has_returncode(1) def testMonoProgress(self): self._testCompactProgress('mono') def testColorProgress(self): self._testCompactProgress('color') def _testCompactProgress(self, name): result = self.run_tests( '--progress=%s' % name, 'sweet/cherries', 'sweet/bananas', infra_staging=False, ) if name == 'color': expected = ('\033[34m% 28\033[0m|' '\033[32m+ 1\033[0m|' '\033[31m- 1\033[0m]: Done') else: expected = '% 28|+ 1|- 1]: Done' result.stdout_includes(expected) result.stdout_includes('sweet/cherries') result.stdout_includes('sweet/bananas') result.has_returncode(1) def testCompactErrorDetails(self): result = self.run_tests( '--progress=mono', 'sweet/cherries', 'sweet/strawberries', infra_staging=False, ) result.stdout_includes('sweet/cherries') result.stdout_includes('sweet/strawberries') result.stdout_includes('+Mock diff') result.has_returncode(1) def testExitAfterNFailures(self): result = self.run_tests( '--progress=verbose', '--exit-after-n-failures=2', '-j1', 'sweet/mangoes', # PASS 'sweet/strawberries', # FAIL 'sweet/blackberries', # FAIL 'sweet/raspberries', # should not run ) result.stdout_includes('sweet/mangoes default: PASS') result.stdout_includes('sweet/strawberries default: FAIL') result.stdout_includes('Too many failures, exiting...') result.stdout_includes('sweet/blackberries default: FAIL') result.stdout_excludes('sweet/raspberries') result.stdout_includes('2 tests failed') result.stdout_includes('3 tests ran') result.has_returncode(1) def testHeavySequence(self): """Test a configuration with 2 heavy tests. One heavy test will get buffered before being run. """ appended = 0 popped = 0 def mock_deque(): class MockQ(): def __init__(self): self.buffer = deque() def append(self, test): nonlocal appended self.buffer.append(test) appended += 1 def popleft(self): nonlocal popped popped += 1 return self.buffer.popleft() def __bool__(self): return bool(self.buffer) return MockQ() # Swarming option will trigger a cleanup routine on the bot def mock_kill(): pass with patch('testrunner.testproc.sequence.deque', side_effect=mock_deque), \ patch('testrunner.testproc.util.kill_processes_linux', side_effect=mock_kill): result = self.run_tests( '--variants=default', '--swarming', 'fat', baseroot="testroot6") result.has_returncode(1) self.assertEqual(1, appended) self.assertEqual(1, popped) def testRunnerFlags(self): """Test that runner-specific flags are passed to tests.""" result = self.run_tests( '--progress=verbose', '--variants=default', '--random-seed=42', 'sweet/bananas', '-v', ) result.stdout_includes( '--test bananas --random-seed=42 --nohard-abort --testing-d8-test-runner') result.has_returncode(0) class FakeTimeoutProc(base.TestProcObserver): """Fake of the total-timeout observer that just stops after counting "count" number of test or result events. """ def __init__(self, count): super(FakeTimeoutProc, self).__init__() self._n = 0 self._count = count def _on_next_test(self, test): self.__on_event() def _on_result_for(self, test, result): self.__on_event() def __on_event(self): if self._n >= self._count: self.stop() self._n += 1 class NumFuzzerTest(TestRunnerTest): def get_runner_class(self): return num_fuzzer.NumFuzzer def testNumFuzzer(self): fuzz_flags = [ f'{flag}=1' for flag in self.get_runner_options() if flag.startswith('--stress-') or flag.startswith('--allocation') ] self.assertEqual(len(fuzz_flags), len(fuzzer.FUZZERS)) for fuzz_flag in fuzz_flags: # The fake timeout observer above will stop after proessing the 10th # test. This still executes an 11th. Each test causes a test- and a # result event internally. We test both paths here. for event_count in (19, 20): with self.subTest(f'fuzz_flag={fuzz_flag} event_count={event_count}'): with patch( 'testrunner.testproc.timeout.TimeoutProc.create', lambda x: FakeTimeoutProc(event_count)): result = self.run_tests( '--command-prefix', sys.executable, '--outdir', 'out/build', '--variants=default', '--fuzzer-random-seed=12345', '--total-timeout-sec=60', fuzz_flag, '--progress=verbose', 'sweet/bananas', ) result.has_returncode(0) result.stdout_includes('>>> Statusfile variables:') result.stdout_includes('11 tests ran') def _run_test_with_random_skip(self, prob): """Run a test root that marks sweet/apples as RARE and pass a probability for random skipping rare tests. """ with patch('testrunner.testproc.timeout.TimeoutProc.create', lambda x: FakeTimeoutProc(10)): return self.run_tests( '--command-prefix', sys.executable, '--outdir', 'out/build', '--variants=default', '--fuzzer-random-seed=12345', '--total-timeout-sec=60', '--allocation-offset=1', '--progress=verbose', f'--skip-rare-tests-prob={prob}', 'sweet', baseroot="testroot8") def testRandomSkip_Includes(self): """Ensure that a test case marked as FUZZ_RARE (here apples) is still included if the probability for skipping is 0. """ result = self._run_test_with_random_skip(0.0) result.has_returncode(1) result.stdout_includes('sweet/apples default: FAIL') result.stdout_includes('sweet/bananas default: PASS') result.stdout_includes('7 tests ran') def testRandomSkip_Excludes(self): """Ensure that a test case marked as FUZZ_RARE (here apples) is always excluded if the probability for skipping is 1. """ result = self._run_test_with_random_skip(1.0) result.has_returncode(0) result.stdout_excludes('sweet/apples') result.stdout_includes('sweet/bananas default: PASS') result.stdout_includes('6 tests ran') class OtherTest(TestRunnerTest): def testStatusFilePresubmit(self): """Test that the fake status file is well-formed.""" with temp_base() as basedir: from testrunner.local import statusfile self.assertTrue(statusfile.PresubmitCheck( basedir / 'test' / 'sweet' / 'sweet.status')) if __name__ == '__main__': unittest.main()