1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
|
#!/usr/bin/env python3
#
# Copyright 2017 the V8 project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import sys
# Adds testrunner to the path hence it has to be imported at the beggining.
from testrunner import base_runner
from testrunner.local import utils
from testrunner.testproc import fuzzer
from testrunner.testproc.combiner import CombinerProc
from testrunner.testproc.execution import ExecutionProc
from testrunner.testproc.expectation import ExpectationProc
from testrunner.testproc.filter import (NameFilterProc, FuzzRareTestFilterProc,
StatusFileFilterProc)
from testrunner.testproc.loader import LoadProc
from testrunner.utils import random_utils
from testrunner.testproc.rerun import RerunProc
from testrunner.testproc.timeout import TimeoutProc
from testrunner.testproc.progress import ResultsTracker, ProgressProc
from testrunner.testproc.shard import ShardProc
DEFAULT_SUITES = ["mjsunit", "webkit", "benchmarks"]
class NumFuzzer(base_runner.BaseTestRunner):
def __init__(self, *args, **kwargs):
super(NumFuzzer, self).__init__(*args, **kwargs)
@property
def default_framework_name(self):
return 'num_fuzzer'
def _add_parser_options(self, parser):
parser.add_option("--fuzzer-random-seed", default=0,
help="Default seed for initializing fuzzer random "
"generator")
parser.add_option("--tests-count", default=5, type="int",
help="Number of tests to generate from each base test. "
"Can be combined with --total-timeout-sec with "
"value 0 to provide infinite number of subtests. "
"When --combine-tests is set it indicates how many "
"tests to create in total")
# Stress gc
parser.add_option(
"--allocation-offset",
default=0,
type="int",
help="probability [0-10] of adding allocation padding "
"to the test")
parser.add_option("--stress-marking", default=0, type="int",
help="probability [0-10] of adding --stress-marking "
"flag to the test")
parser.add_option("--stress-scavenge", default=0, type="int",
help="probability [0-10] of adding --stress-scavenge "
"flag to the test")
parser.add_option("--stress-compaction", default=0, type="int",
help="probability [0-10] of adding --stress-compaction "
"flag to the test")
parser.add_option("--stress-gc", default=0, type="int",
help="probability [0-10] of adding --random-gc-interval "
"flag to the test")
# Stress stack size
parser.add_option("--stress-stack-size", default=0, type="int",
help="probability [0-10] of adding --stack-size "
"flag to the test")
# Stress tasks
parser.add_option("--stress-delay-tasks", default=0, type="int",
help="probability [0-10] of adding --stress-delay-tasks "
"flag to the test")
parser.add_option("--stress-thread-pool-size", default=0, type="int",
help="probability [0-10] of adding --thread-pool-size "
"flag to the test")
# Stress compiler
parser.add_option("--stress-deopt", default=0, type="int",
help="probability [0-10] of adding --deopt-every-n-times "
"flag to the test")
parser.add_option("--stress-interrupt-budget", default=0, type="int",
help="probability [0-10] of adding the --interrupt-budget "
"flag to the test")
# Combine multiple tests
parser.add_option("--combine-tests", default=False, action="store_true",
help="Combine multiple tests as one and run with "
"try-catch wrapper")
parser.add_option("--combine-max", default=100, type="int",
help="Maximum number of tests to combine")
parser.add_option("--combine-min", default=2, type="int",
help="Minimum number of tests to combine")
# Miscellaneous
parser.add_option("--variants", default='default',
help="Comma-separated list of testing variants")
parser.add_option(
"--skip-rare-tests-prob",
default=0.75,
type="float",
help="probability in [0.0, 1.0] of skipping tests "
"marked as RARE")
return parser
def _process_options(self):
if not self.options.fuzzer_random_seed:
self.options.fuzzer_random_seed = random_utils.random_seed()
if self.options.total_timeout_sec:
self.options.tests_count = 0
if self.options.combine_tests:
if self.options.combine_min > self.options.combine_max:
print(('min_group_size (%d) cannot be larger than max_group_size (%d)' %
self.options.min_group_size, self.options.max_group_size))
raise base_runner.TestRunnerError()
if self.options.variants != 'default':
print ('Only default testing variant is supported with numfuzz')
raise base_runner.TestRunnerError()
return True
def _get_default_suite_names(self):
return DEFAULT_SUITES
def _get_statusfile_variables(self, context):
variables = (
super(NumFuzzer, self)._get_statusfile_variables(context))
variables.update({
'deopt_fuzzer':
bool(self.options.stress_deopt),
'interrupt_fuzzer':
bool(self.options.stress_interrupt_budget),
'endurance_fuzzer':
bool(self.options.combine_tests),
'gc_stress':
bool(self.options.stress_gc),
'gc_fuzzer':
bool(
max([
self.options.allocation_offset, self.options.stress_marking,
self.options.stress_scavenge,
self.options.stress_compaction, self.options.stress_gc,
self.options.stress_delay_tasks,
self.options.stress_stack_size,
self.options.stress_thread_pool_size
])),
})
return variables
def _create_test_config(self):
# Multiple stress flags together make tests run much longer.
config = super()._create_test_config()
config.timeout *= 2
return config
def _do_execute(self, tests, args, ctx):
loader = LoadProc(tests)
combiner = CombinerProc.create(self.options)
results = ResultsTracker.create(self.options)
execproc = ExecutionProc(ctx, self.options.j)
sigproc = self._create_signal_proc()
progress = ProgressProc(ctx, self.options, tests.test_count_estimate)
procs = [
loader,
NameFilterProc(args) if args else None,
StatusFileFilterProc(None, None),
FuzzRareTestFilterProc(self.options.fuzzer_rng(),
self.options.skip_rare_tests_prob),
# TODO(majeski): Improve sharding when combiner is present. Maybe select
# different random seeds for shards instead of splitting tests.
ShardProc.create(self.options),
ExpectationProc(),
combiner,
fuzzer.FuzzerProc.create(self.options),
sigproc,
progress,
results,
TimeoutProc.create(self.options),
RerunProc.create(self.options),
execproc,
]
procs = [p for p in procs if p]
self._prepare_procs(procs)
loader.load_initial_tests()
# TODO(majeski): maybe some notification from loader would be better?
if combiner:
combiner.generate_initial_tests(self.options.j * 4)
# This starts up worker processes and blocks until all tests are
# processed.
execproc.run()
progress.finished()
print('>>> %d tests ran' % results.total)
if results.failed:
return utils.EXIT_CODE_FAILURES
# Indicate if a SIGINT or SIGTERM happened.
return sigproc.exit_code
def _is_testsuite_supported(self, suite):
return not self.options.combine_tests or suite.test_combiner_available()
if __name__ == '__main__':
sys.exit(NumFuzzer().execute()) # pragma: no cover
|
