1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
|
# Copyright 2019 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import os
import shutil
import tempfile
import unittest
from unittest import mock
from telemetry import decorators
from telemetry.testing import test_stories
from telemetry.web_perf import timeline_based_measurement
from tracing.value.diagnostics import all_diagnostics
from tracing.value.diagnostics import reserved_infos
from tracing.value import histogram_set
from core import benchmark_runner
from core import perf_benchmark
from core import results_processor
from core import testing
def _FakeParseArgs(environment, args, results_arg_parser):
del environment # Unused.
options, _ = results_arg_parser.parse_known_args(args)
return options
class BenchmarkRunnerUnittest(unittest.TestCase):
"""Tests for benchmark_runner.main which mock out benchmark running."""
def testMain_ReturnCode(self):
"""Test that benchmark_runner.main() respects return code from Telemetry."""
config = mock.Mock()
with mock.patch('core.benchmark_runner.command_line') as telemetry_cli:
telemetry_cli.ParseArgs.side_effect = _FakeParseArgs
telemetry_cli.RunCommand.return_value = 42
# Note: We pass `--output-format none` and a non-existent output
# dir to prevent the results processor from processing any results.
return_code = benchmark_runner.main(config, [
'run', 'some.benchmark', '--browser', 'stable',
'--output-dir', '/does/not/exist', '--output-format', 'none'])
self.assertEqual(return_code, 42)
class BenchmarkRunnerIntegrationTest(unittest.TestCase):
"""Integration tests for benchmark running and results processing.
Note, however, no command line processing is tested.
"""
def setUp(self):
self.options = testing.GetRunOptions(
output_dir=tempfile.mkdtemp())
self.options.output_formats = ['histograms']
results_processor.ProcessOptions(self.options)
def tearDown(self):
shutil.rmtree(self.options.output_dir)
def assertHasDiagnostic(self, hist, diag_info, value=None):
"""Assert that a histogram is associated with the given diagnostic."""
self.assertIn(diag_info.name, hist.diagnostics)
diag = hist.diagnostics[diag_info.name]
self.assertIsInstance(
diag, all_diagnostics.GetDiagnosticClassForName(diag_info.type))
if value is not None:
# Assume we expect singleton GenericSet with the given value.
self.assertEqual(len(diag), 1)
self.assertEqual(next(iter(diag)), value)
def RunBenchmark(self, benchmark_class):
"""Run a benchmark, process results, and return generated histograms."""
# TODO(crbug.com/40636798): Ideally we should be able to just call
# telemetry.command_line.RunCommand(self.options) with the right set
# of options chosen. However, argument parsing and command running are
# currently tangled in Telemetry. In particular the class property
# Run._benchmark is not set when we skip argument parsing, and the Run.Run
# method call fails. Simplify this when argument parsing and command
# running are no longer intertwined like this.
run_return_code = benchmark_class().Run(self.options)
self.assertEqual(run_return_code, 0)
process_return_code = results_processor.ProcessResults(self.options,
is_unittest=True)
self.assertEqual(process_return_code, 0)
histograms_file = os.path.join(self.options.output_dir, 'histograms.json')
self.assertTrue(os.path.exists(histograms_file))
with open(histograms_file) as f:
dicts = json.load(f)
histograms = histogram_set.HistogramSet()
histograms.ImportDicts(dicts)
return histograms
@decorators.Disabled(
'chromeos', # TODO(crbug.com/40137013): Fix the test.
'android-nougat', # Flaky: https://crbug.com/1342706
'mac') # Failing: https://crbug.com/1370958
def testTimelineBasedEndToEnd(self):
class TestTimelineBasedBenchmark(perf_benchmark.PerfBenchmark):
"""A dummy benchmark that records a trace and runs sampleMetric on it."""
def CreateCoreTimelineBasedMeasurementOptions(self):
options = timeline_based_measurement.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetrics(['consoleErrorMetric'])
return options
def CreateStorySet(self, _):
def log_error(action_runner):
action_runner.EvaluateJavaScript('console.error("foo!")')
return test_stories.SinglePageStorySet(
name='log_error_story', story_run_side_effect=log_error)
@classmethod
def Name(cls):
return 'test_benchmark.timeline_based'
histograms = self.RunBenchmark(TestTimelineBasedBenchmark)
# Verify that the injected console.log error was counted by the metric.
hist = histograms.GetHistogramNamed('console:error:js')
self.assertEqual(hist.average, 1)
self.assertHasDiagnostic(hist, reserved_infos.BENCHMARKS,
'test_benchmark.timeline_based')
self.assertHasDiagnostic(hist, reserved_infos.STORIES, 'log_error_story')
self.assertHasDiagnostic(hist, reserved_infos.STORYSET_REPEATS, 0)
self.assertHasDiagnostic(hist, reserved_infos.TRACE_START)
|
