1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
|
# Copyright 2014 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Run the first page of one benchmark for every module.
Only benchmarks that have a composable measurement are included.
Ideally this test would be comprehensive, however, running one page
of every benchmark would run impractically long.
"""
import os
import sys
import unittest
from chrome_telemetry_build import chromium_config
from core import results_processor
from core import testing
from telemetry import benchmark as benchmark_module
from telemetry import decorators
from telemetry.testing import progress_reporter
from py_utils import discover
from py_utils import tempfile_ext
from benchmarks import jetstream2
from benchmarks import octane
from benchmarks import rasterize_and_record_micro
from benchmarks import speedometer1
from benchmarks import v8_browsing
# We want to prevent benchmarks from accidentally trying to upload too much
# data to the chrome perf dashboard. So the smoke tests below cap the max
# number of values that each story tested would produce when running on the
# waterfall.
MAX_VALUES_PER_TEST_CASE = 1000
def SmokeTestGenerator(benchmark_class, num_pages=1, story_tag_filter=None):
"""Generates a smoke test for the first N pages from a benchmark.
Args:
benchmark_class: a benchmark class to smoke test.
num_pages: only smoke test the first N pages, since smoke testing
everything would take too long to run.
story_tag_filter: only smoke test stories matching with tags.
"""
# NOTE TO SHERIFFS: DO NOT DISABLE THIS TEST.
#
# This smoke test dynamically tests all benchmarks. So disabling it for one
# failing or flaky benchmark would disable a much wider swath of coverage
# than is usually intended. Instead, if a particular benchmark is failing,
# disable it in tools/perf/benchmarks/*.
@decorators.Disabled('android') # crbug.com/641934
def BenchmarkSmokeTestFunc(self):
# Some benchmarks are running multiple iterations
# which is not needed for a smoke test
if hasattr(benchmark_class, 'enable_smoke_test_mode'):
benchmark_class.enable_smoke_test_mode = True
with tempfile_ext.NamedTemporaryDirectory() as temp_dir:
options = testing.GetRunOptions(
output_dir=temp_dir,
benchmark_cls=benchmark_class,
overrides={
'story_shard_end_index': num_pages,
'story_tag_filter': story_tag_filter
},
environment=chromium_config.GetDefaultChromiumConfig())
options.pageset_repeat = 1 # For smoke testing only run the page once.
options.output_formats = ['histograms']
options.max_values_per_test_case = MAX_VALUES_PER_TEST_CASE
results_processor.ProcessOptions(options)
return_code = benchmark_class().Run(options)
# TODO(crbug.com/40105219): Make 111 be the exit code that means
# "no stories were run.".
if return_code in (-1, 111):
self.skipTest('The benchmark was not run.')
self.assertEqual(
return_code, 0,
msg='Benchmark run failed: %s' % benchmark_class.Name())
return_code = results_processor.ProcessResults(options, is_unittest=True)
self.assertEqual(
return_code, 0,
msg='Result processing failed: %s' % benchmark_class.Name())
# Set real_test_func as benchmark_class to make typ
# write benchmark_class source filepath to trace instead of
# path to this file
BenchmarkSmokeTestFunc.real_test_func = benchmark_class
return BenchmarkSmokeTestFunc
# The list of benchmark modules to be excluded from our smoke tests.
_BLOCK_LIST_TEST_MODULES = {
octane, # Often fails & take long time to timeout on cq bot.
rasterize_and_record_micro, # Always fails on cq bot.
speedometer1, # Takes 101 seconds.
jetstream2, # Causes CQ shard to timeout, crbug.com/992837
v8_browsing, # Flaky on Android, crbug.com/628368.
}
# The list of benchmark names to be excluded from our smoke tests.
_BLOCK_LIST_TEST_NAMES = [
'memory.long_running_idle_gmail_background_tbmv2',
'UNSCHEDULED_ad_frames.iframe', # b/342449133
'UNSCHEDULED_oortonline_tbmv2',
'webrtc', # crbug.com/932036
'v8.runtime_stats.top_25', # Fails in Windows, crbug.com/1043048
'wasmpspdfkit', # Fails in Chrome OS, crbug.com/1191938
'memory.desktop', # crbug.com/1277277 and b/286898261
'desktop_ui' if sys.platform == 'darwin' else None, # crbug.com/1370958
'power.desktop' if sys.platform == 'darwin' else None, # crbug.com/1370958
]
def MergeDecorators(method, method_attribute, benchmark, benchmark_attribute):
# Do set union of attributes to eliminate duplicates.
merged_attributes = getattr(method, method_attribute, set()).union(
getattr(benchmark, benchmark_attribute, set()))
if merged_attributes:
setattr(method, method_attribute, merged_attributes)
class BenchmarkSmokeTest(unittest.TestCase):
pass
def load_tests(loader, standard_tests, pattern):
del loader, standard_tests, pattern # unused
suite = progress_reporter.TestSuite()
benchmarks_dir = os.path.dirname(__file__)
top_level_dir = os.path.dirname(benchmarks_dir)
# Using the default of |index_by_class_name=False| means that if a module
# has multiple benchmarks, only the last one is returned.
all_benchmarks = discover.DiscoverClasses(
benchmarks_dir, top_level_dir, benchmark_module.Benchmark,
index_by_class_name=False).values()
for benchmark in all_benchmarks:
if sys.modules[benchmark.__module__] in _BLOCK_LIST_TEST_MODULES:
continue
if benchmark.Name() in _BLOCK_LIST_TEST_NAMES:
continue
if 'desktop_ui' in benchmark.Name():
# Run tests with a specific smoke_test tag.
method = SmokeTestGenerator(benchmark,
num_pages=None,
story_tag_filter='smoke_test')
else:
method = SmokeTestGenerator(benchmark)
# Make sure any decorators are propagated from the original declaration.
# (access to protected members) pylint: disable=protected-access
# TODO(dpranke): Since we only pick the first test from every class
# (above), if that test is disabled, we'll end up not running *any*
# test from the class. We should probably discover all of the tests
# in a class, and then throw the ones we don't need away instead.
disabled_benchmark_attr = decorators.DisabledAttributeName(benchmark)
disabled_method_attr = decorators.DisabledAttributeName(method)
enabled_benchmark_attr = decorators.EnabledAttributeName(benchmark)
enabled_method_attr = decorators.EnabledAttributeName(method)
MergeDecorators(method, disabled_method_attr, benchmark,
disabled_benchmark_attr)
MergeDecorators(method, enabled_method_attr, benchmark,
enabled_benchmark_attr)
setattr(BenchmarkSmokeTest, benchmark.Name(), method)
suite.addTest(BenchmarkSmokeTest(benchmark.Name()))
return suite
|
