1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
|
# Copyright 2017 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import logging
import sys
from core import perf_benchmark
from core import platforms as core_platforms
import page_sets
from page_sets.system_health import platforms
from telemetry import benchmark
from telemetry import story as story_module
from telemetry.timeline import chrome_trace_category_filter
from telemetry.web_perf import timeline_based_measurement
RENDERING_BENCHMARK_UMA = [
'Compositing.Display.DrawToSwapUs',
'CompositorLatency.TotalLatency',
'EventLatency.FirstGestureScrollUpdate.TotalLatency2',
'EventLatency.GestureScrollUpdate.Touchscreen.TotalLatency',
'EventLatency.GestureScrollUpdate.TotalLatency2',
'Graphics.Smoothness.Checkerboarding4.AllAnimations',
'Graphics.Smoothness.Checkerboarding4.AllInteractions',
'Graphics.Smoothness.Checkerboarding4.AllSequences',
'Graphics.Smoothness.CheckerboardingNeedRaster4.AllSequences',
'Graphics.Smoothness.CheckerboardingNeedRecord4.AllSequences',
'Graphics.Smoothness.Jank3.AllAnimations',
'Graphics.Smoothness.Jank3.AllInteractions',
'Graphics.Smoothness.Jank3.AllSequences',
'Graphics.Smoothness.PercentDroppedFrames3.AllAnimations',
'Graphics.Smoothness.PercentDroppedFrames3.AllInteractions',
'Graphics.Smoothness.PercentDroppedFrames3.AllSequences',
'Memory.GPU.PeakMemoryUsage2.Scroll',
'Memory.GPU.PeakMemoryUsage2.PageLoad',
'Event.Jank.PredictorJankyFramePercentage2',
'Event.ScrollJank.DelayedFramesPercentage.FixedWindow',
'Event.ScrollJank.DelayedFramesPercentage.PerScroll',
'Event.ScrollJank.MissedVsyncsSum.FixedWindow',
'Event.ScrollJank.MissedVsyncsSum.PerScroll',
'Event.ScrollJank.MissedVsyncsPercentage.FixedWindow',
'Event.ScrollJank.MissedVsyncsPercentage.PerScroll',
]
class _RenderingBenchmark(perf_benchmark.PerfBenchmark):
# TODO(crbug.com/40764818): Capturing video is causing long cycle time and timeout
# on some Pixel devices. Disabling this option until the issue can be fixed.
#options = {
# 'capture_screen_video': True
#}
@classmethod
def AddBenchmarkCommandLineArgs(cls, parser):
parser.add_argument(
'--scroll-forever',
action='store_true',
help=('If set, continuously scroll up and down forever. '
'This is useful for analysing scrolling behaviour '
'with tools such as perf.'))
parser.add_argument(
'--allow-software-compositing',
action='store_true',
help=('If set, allows the benchmark to run with software '
'compositing.'))
parser.add_argument(
'--extra-uma-metrics',
help=('Comma separated list of additional UMA metrics to '
'include in result output. Note that histogram buckets '
'in telemetry report may not match buckets from UMA.'))
@classmethod
def ProcessCommandLineArgs(cls, parser, args):
cls.allow_software_compositing = args.allow_software_compositing
cls.uma_metrics = RENDERING_BENCHMARK_UMA
if args.extra_uma_metrics:
cls.uma_metrics += args.extra_uma_metrics.split(',')
def CreateStorySet(self, options):
return page_sets.RenderingStorySet(platform=self.PLATFORM_NAME)
def SetExtraBrowserOptions(self, options):
options.AppendExtraBrowserArgs('--enable-gpu-benchmarking')
options.AppendExtraBrowserArgs('--touch-events=enabled')
# TODO(jonross): Catapult's record_wpr.py calls SetExtraBrowserOptions
# before calling ProcessCommandLineArgs. This will crash attempting to
# record new rendering benchmarks. We do not want to support software
# compositing for recording, so for now we will just check for the existence
# the flag. We will review updating Catapult at a later point.
if (hasattr(self, 'allow_software_compositing')
and self.allow_software_compositing) or self.NeedsSoftwareCompositing():
logging.warning('Allowing software compositing. Some of the reported '
'metrics will have unreliable values.')
else:
options.AppendExtraBrowserArgs('--disable-software-compositing-fallback')
def CreateCoreTimelineBasedMeasurementOptions(self):
category_filter = chrome_trace_category_filter.CreateLowOverheadFilter()
category_filter.AddDisabledByDefault(
'disabled-by-default-histogram_samples')
options = timeline_based_measurement.Options(category_filter)
options.config.chrome_trace_config.EnableUMAHistograms(*self.uma_metrics)
options.SetTimelineBasedMetrics([
'renderingMetric',
'umaMetric',
# Unless --experimentatil-tbmv3-metric flag is used, the following tbmv3
# metrics do nothing.
'tbmv3:uma_metrics'
])
return options
@benchmark.Info(
emails=['jonross@chromium.org', 'chrome-gpu-metrics@google.com'],
documentation_url='https://bit.ly/rendering-benchmarks',
component='Internals>GPU>Metrics')
class RenderingDesktop(_RenderingBenchmark):
# TODO(johnchen): Remove either the SUPPORTED_PLATFORMS or
# SUPPORTED_PLATFORMS_TAGS lists. Only one is necessary.
SUPPORTED_PLATFORMS = [story_module.expectations.ALL_DESKTOP]
SUPPORTED_PLATFORM_TAGS = [core_platforms.DESKTOP]
PLATFORM_NAME = platforms.DESKTOP
@classmethod
def Name(cls):
return 'rendering.desktop'
def SetExtraBrowserOptions(self, options):
super(RenderingDesktop, self).SetExtraBrowserOptions(options)
if sys.platform == 'darwin':
# Mac bots without a physical display fallbacks to SRGB. This flag forces
# them to use a color profile (P3), which matches the usual color profile
# on Mac monitors and changes the cost of some overlay operations to match
# real conditions more closely.
options.AppendExtraBrowserArgs('--force-color-profile=display-p3-d65')
@benchmark.Info(
emails=['jonross@chromium.org', 'chrome-gpu-metrics@google.com'],
documentation_url='https://bit.ly/rendering-benchmarks',
component='Internals>GPU>Metrics')
class RenderingDesktopNoTracing(RenderingDesktop):
@classmethod
def Name(cls):
return 'rendering.desktop.notracing'
def CreateStorySet(self, options):
os_name = None
# Archive Validation does not perform OS validation
if hasattr(options, 'os_name'):
os_name = options.os_name
return page_sets.RenderingStorySet(platform=self.PLATFORM_NAME,
disable_tracing=True,
os_name=os_name)
def CreateCoreTimelineBasedMeasurementOptions(self):
options = timeline_based_measurement.Options()
options.config.enable_chrome_trace = False
options.config.enable_platform_display_trace = False
return options
@benchmark.Info(
emails=['jonross@chromium.org', 'chrome-gpu-metrics@google.com'],
documentation_url='https://bit.ly/rendering-benchmarks',
component='Internals>GPU>Metrics')
class RenderingMobile(_RenderingBenchmark):
# TODO(johnchen): Remove either the SUPPORTED_PLATFORMS or
# SUPPORTED_PLATFORMS_TAGS lists. Only one is necessary.
SUPPORTED_PLATFORMS = [
story_module.expectations.ALL_MOBILE,
story_module.expectations.FUCHSIA_ASTRO,
story_module.expectations.FUCHSIA_SHERLOCK
]
SUPPORTED_PLATFORM_TAGS = [
core_platforms.MOBILE, core_platforms.FUCHSIA_ASTRO,
core_platforms.FUCHSIA_SHERLOCK
]
PLATFORM_NAME = platforms.MOBILE
@classmethod
def Name(cls):
return 'rendering.mobile'
def SetExtraBrowserOptions(self, options):
super(RenderingMobile, self).SetExtraBrowserOptions(options)
# Disable locking the controls as visible for a minimum duration. This
# allows controls to unlock after page load, rather than in the middle of a
# story.
options.AppendExtraBrowserArgs('--disable-minimum-show-duration')
# Force online state for the offline indicator so it doesn't show and affect
# the benchmarks on bots, which are offline by default.
options.AppendExtraBrowserArgs(
'--force-online-connection-state-for-indicator')
def CreateCoreTimelineBasedMeasurementOptions(self):
options = super(
RenderingMobile, self).CreateCoreTimelineBasedMeasurementOptions()
options.config.enable_platform_display_trace = True
return options
@benchmark.Info(
emails=['jonross@chromium.org', 'chrome-gpu-metrics@google.com'],
documentation_url='https://bit.ly/rendering-benchmarks',
component='Internals>GPU>Metrics')
class RenderingMobileNoTracing(RenderingMobile):
@classmethod
def Name(cls):
return 'rendering.mobile.notracing'
def CreateStorySet(self, options):
return page_sets.RenderingStorySet(platform=self.PLATFORM_NAME,
disable_tracing=True)
def CreateCoreTimelineBasedMeasurementOptions(self):
options = timeline_based_measurement.Options()
options.config.enable_chrome_trace = False
options.config.enable_platform_display_trace = False
return options
|
