File: minidump_unittest.py

package info (click to toggle)
chromium 138.0.7204.157-1
  • links: PTS, VCS
  • area: main
  • in suites: trixie
  • size: 6,071,864 kB
  • sloc: cpp: 34,936,859; ansic: 7,176,967; javascript: 4,110,704; python: 1,419,953; asm: 946,768; xml: 739,967; pascal: 187,324; sh: 89,623; perl: 88,663; objc: 79,944; sql: 50,304; cs: 41,786; fortran: 24,137; makefile: 21,806; php: 13,980; tcl: 13,166; yacc: 8,925; ruby: 7,485; awk: 3,720; lisp: 3,096; lex: 1,327; ada: 727; jsp: 228; sed: 36
file content (303 lines) | stat: -rw-r--r-- 13,840 bytes parent folder | download | duplicates (3)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
# Copyright 2015 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import logging
import os
import sys
import time

from telemetry.core import exceptions
from telemetry.internal.results import artifact_compatibility_wrapper as acw
from telemetry.internal.results import artifact_logger
from telemetry.testing import tab_test_case
from telemetry import decorators

import py_utils


# Possible ways that gl::Crash() will show up in a stack trace.
GPU_CRASH_SIGNATURES = [
    'gl::Crash',
    'chrome!Crash',
    'GpuServiceImpl::Crash()',
]
# Possible ways that a renderer process crash intentionally caused by DevTools
# can show up in a stack trace.
FORCED_RENDERER_CRASH_SIGNATURES = [
    'base::debug::BreakDebugger',
    'blink::DevToolsSession::IOSession::DispatchProtocolCommand',
    'blink::HandleChromeDebugURL',
    'chrome!DispatchProtocolCommand',
    'logging::LogMessage::~LogMessage',
]

# At the time of writing, this is the default timeout for
# App.GetRecentMinidumpPathWithTimeout(). Ideally, we would be able to pass in
# None, but that will require updating Telemetry.
WAIT_FOR_MINIDUMP_TIMEOUT = 15
if sys.platform == 'win32':
  # TODO(crbug.com/406190893): Remove this special case if the cause of slow
  # minidump generation on Windows is fixed.
  WAIT_FOR_MINIDUMP_TIMEOUT = 30


def ContainsAtLeastOne(expected_values, checked_value):
  for expected in expected_values:
    if expected in checked_value:
      return True
  return False


class BrowserMinidumpTest(tab_test_case.TabTestCase):
  def setUp(self):
    # If something is wrong with minidump symbolization, we want to get all the
    # debugging information we can from the bots since it may be difficult to
    # reproduce the issue locally. So, use the full logger implementation.
    artifact_logger.RegisterArtifactImplementation(
        acw.FullLoggingArtifactImpl())
    super(BrowserMinidumpTest, self).setUp()

  def tearDown(self):
    super(BrowserMinidumpTest, self).tearDown()
    artifact_logger.RegisterArtifactImplementation(None)

  def assertContainsAtLeastOne(self, expected_values, checked_value):
    self.assertTrue(ContainsAtLeastOne(expected_values, checked_value),
                    'None of %s found in %s' % (expected_values, checked_value))

  @decorators.Isolated
  # Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
  # still read-only, so skip the test in that case.
  @decorators.Disabled(
      'chromeos-local',
      'win7',  # https://crbug.com/1084931
  )
  def testSymbolizeMinidump(self):
    # Wait for the browser to restart fully before crashing
    self._LoadPageThenWait('var sam = "car";', 'sam')
    self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
    crash_minidump_path = self._browser.GetRecentMinidumpPathWithTimeout(
        timeout_s=WAIT_FOR_MINIDUMP_TIMEOUT)
    self.assertIsNotNone(crash_minidump_path)

    if crash_minidump_path is not None:
      logging.info('testSymbolizeMinidump: most recent path = '
          + crash_minidump_path)
    all_paths = self._browser.GetAllMinidumpPaths()
    if all_paths is not None:
      logging.info('testSymbolizeMinidump: all paths ' + ''.join(all_paths))
    all_unsymbolized_paths = self._browser.GetAllUnsymbolizedMinidumpPaths()
    if all_unsymbolized_paths is not None:
      logging.info('testSymbolizeMinidump: all unsymbolized paths '
          + ''.join(all_unsymbolized_paths))

    # Flakes on chromeos: crbug.com/1014754
    # This has failed to repro either locally or on swarming, so dump extra
    # information if this is hit on the bots.
    if len(all_unsymbolized_paths) != 1:
      self._browser.CollectDebugData(logging.ERROR)
    self.assertTrue(len(all_unsymbolized_paths) == 1)

    # Now symbolize that minidump and make sure there are no longer any present
    succeeded, stack = self._browser.SymbolizeMinidump(crash_minidump_path)
    self.assertTrue(succeeded)
    self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)

    all_unsymbolized_after_symbolize_paths = \
        self._browser.GetAllUnsymbolizedMinidumpPaths()
    if all_unsymbolized_after_symbolize_paths is not None:
      logging.info('testSymbolizeMinidump: after symbolize all '
          + 'unsymbolized paths: '
          + ''.join(all_unsymbolized_after_symbolize_paths))
    self.assertTrue(len(all_unsymbolized_after_symbolize_paths) == 0)

  @decorators.Isolated
  # Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
  # still read-only, so skip the test in that case.
  @decorators.Disabled(
      'chromeos-local',
      'win7',  # https://crbug.com/1084931
  )
  def testMultipleCrashMinidumps(self):
    # Wait for the browser to restart fully before crashing
    self._LoadPageThenWait('var cat = "dog";', 'cat')
    self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
    first_crash_path = self._browser.GetRecentMinidumpPathWithTimeout(
        timeout_s=WAIT_FOR_MINIDUMP_TIMEOUT)

    self.assertIsNotNone(first_crash_path)
    if first_crash_path is not None:
      logging.info('testMultipleCrashMinidumps: first crash most recent path ' +
                   first_crash_path)
    all_paths = self._browser.GetAllMinidumpPaths()
    if all_paths is not None:
      logging.info('testMultipleCrashMinidumps: first crash all paths: '
          + ''.join(all_paths))
    # Flakes on chromeos: crbug.com/1014754
    # This has failed to repro either locally or on swarming, so dump extra
    # information if this is hit on the bots.
    if len(all_paths) != 1:
      self._browser.CollectDebugData(logging.ERROR)
    self.assertEqual(len(all_paths), 1)
    self.assertEqual(all_paths[0], first_crash_path)
    all_unsymbolized_paths = self._browser.GetAllUnsymbolizedMinidumpPaths()
    self.assertTrue(len(all_unsymbolized_paths) == 1)
    if all_unsymbolized_paths is not None:
      logging.info('testMultipleCrashMinidumps: first crash all unsymbolized '
          'paths: ' + ''.join(all_unsymbolized_paths))

    # Restart the browser and then crash a second time
    logging.info('Restarting the browser')
    self._RestartBrowser()

    # Start a new tab in the restarted browser
    self._LoadPageThenWait('var foo = "bar";', 'foo')

    self._browser.tabs.New().Navigate('chrome://gpucrash', timeout=10)
    # Make the oldest allowable timestamp slightly after the first dump's
    # timestamp so we don't get the first one returned to us again
    oldest_ts = os.path.getmtime(first_crash_path) + 1
    second_crash_path = self._browser.GetRecentMinidumpPathWithTimeout(
        timeout_s=WAIT_FOR_MINIDUMP_TIMEOUT, oldest_ts=oldest_ts)
    self.assertIsNotNone(second_crash_path)
    if second_crash_path is not None:
      logging.info(
          'testMultipleCrashMinidumps: second crash most recent path ' +
          second_crash_path)
    second_crash_all_paths = self._browser.GetAllMinidumpPaths()
    if second_crash_all_paths is not None:
      logging.info('testMultipleCrashMinidumps: second crash all paths: '
          + ''.join(second_crash_all_paths))
    second_crash_all_unsymbolized_paths = \
        self._browser.GetAllUnsymbolizedMinidumpPaths()
    #self.assertTrue(len(all_unsymbolized_paths) == 1)
    if second_crash_all_unsymbolized_paths is not None:
      logging.info('testMultipleCrashMinidumps: second crash all unsymbolized '
          'paths: ' + ''.join(second_crash_all_unsymbolized_paths))
    self.assertEqual(len(second_crash_all_paths), 2)
    # Check that both paths are now present and unsymbolized
    self.assertTrue(first_crash_path in second_crash_all_paths)
    self.assertTrue(second_crash_path in second_crash_all_paths)
    self.assertTrue(len(second_crash_all_unsymbolized_paths) == 2)


    # Now symbolize one of those paths and assert that there is still one
    # unsymbolized
    succeeded, stack = self._browser.SymbolizeMinidump(second_crash_path)
    self.assertTrue(succeeded)
    self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)

    after_symbolize_all_paths = self._browser.GetAllMinidumpPaths()
    if after_symbolize_all_paths is not None:
      logging.info('testMultipleCrashMinidumps: after symbolize all paths: '
          + ''.join(after_symbolize_all_paths))
    self.assertEqual(len(after_symbolize_all_paths), 2)
    after_symbolize_all_unsymbolized_paths = \
        self._browser.GetAllUnsymbolizedMinidumpPaths()
    if after_symbolize_all_unsymbolized_paths is not None:
      logging.info('testMultipleCrashMinidumps: after symbolize all '
          + 'unsymbolized paths: '
          + ''.join(after_symbolize_all_unsymbolized_paths))
    self.assertEqual(after_symbolize_all_unsymbolized_paths, [first_crash_path])

    # Explicitly ignore the remaining minidump so that it isn't detected during
    # teardown by the test runner.
    self._browser.IgnoreMinidump(first_crash_path)

  @decorators.Isolated
  # Minidump symbolization doesn't work in ChromeOS local mode if the rootfs is
  # still read-only, so skip the test in that case.
  @decorators.Disabled(
      'chromeos-board-eve',  # b/312565719
      'chromeos-local',
      'win7',  # https://crbug.com/1084931
  )
  def testMinidumpFromRendererHang(self):
    """Tests that renderer hangs result in minidumps.

    Telemetry has logic for detecting renderer hangs and killing the renderer
    and GPU processes in such cases so we can get minidumps for diagnosing the
    root cause.
    """
    self._LoadPageThenWait('var cat = "dog";', 'cat')
    try:
      self._browser.tabs[-1].Navigate('chrome://hang', timeout=10)
    except exceptions.Error:
      # We expect the navigate to time out due to the hang.
      pass
    found_minidumps = False
    try:
      # Hung renderers are detected by JavaScript evaluation timing out, so
      # try to evaluate something to trigger that.
      # The timeout provided is the same one used for crashing the processes, so
      # don't make it too short.
      self._browser.tabs[-1].EvaluateJavaScript('var cat = "dog";', timeout=10)
    except exceptions.TimeoutException:
      # Try to until at least one minidump is written to disk. If none end up
      # being written, the test will fail shortly after.
      _ = self._browser.GetRecentMinidumpPathWithTimeout(
          timeout_s=WAIT_FOR_MINIDUMP_TIMEOUT)

      # If we time out while crashing the renderer process, the minidump should
      # still exist, we just have to manually look for it instead of it being
      # part of the exception.
      all_paths = self._browser.GetAllMinidumpPaths()
      # We can't assert that we have exactly two minidumps because we can also
      # get one from the renderer process being notified of the GPU process
      # crash.
      num_paths = len(all_paths)
      self.assertTrue(num_paths in (2, 3),
                      'Got %d minidumps, expected 2 or 3' % num_paths)
      found_renderer = False
      found_gpu = False
      for p in all_paths:
        succeeded, stack = self._browser.SymbolizeMinidump(p)
        self.assertTrue(succeeded)
        try:
          self.assertContainsAtLeastOne(FORCED_RENDERER_CRASH_SIGNATURES, stack)
          # We don't assert that we haven't found a renderer crash yet since
          # we can potentially get multiple under normal circumstances.
          found_renderer = True
        except AssertionError:
          self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
          self.assertFalse(found_gpu, 'Found two GPU crashes')
          found_gpu = True
      self.assertTrue(found_renderer and found_gpu)
      found_minidumps = True
    except exceptions.AppCrashException as e:
      self.assertTrue(e.is_valid_dump)
      # We should get one minidump from the GPU process (gl::Crash()) and one
      # minidump from the renderer process (base::debug::BreakDebugger()).
      self.assertContainsAtLeastOne(FORCED_RENDERER_CRASH_SIGNATURES,
                                    '\n'.join(e.stack_trace))
      # There appears to be a bug on older versions of Windows 10 where the GPU
      # minidump won't be found by the AppCrashException no matter how long we
      # wait after the crash takes place. So, look for it afterwards.
      if not ContainsAtLeastOne(GPU_CRASH_SIGNATURES, '\n'.join(e.stack_trace)):
        self.assertEqual(sys.platform, 'win32')
        minidumps = self._browser.GetAllUnsymbolizedMinidumpPaths()
        self.assertEqual(len(minidumps), 1)
        succeeded, stack = self._browser.SymbolizeMinidump(minidumps[0])
        self.assertTrue(succeeded)
        self.assertContainsAtLeastOne(GPU_CRASH_SIGNATURES, stack)
      found_minidumps = True
    self.assertTrue(found_minidumps)

  def _LoadPageThenWait(self, script, value):
    # We are occasionally seeing these tests fail on the first load and
    # call to GetMostRecentMinidumpPath, where the directory is coming up empty.
    # We are hypothesizing, that although the browser is technically loaded,
    # some of chromes optimizations could still be running in the background
    # that potentially initializing the crash directory that we set with the
    # environment vairable BREAKPAD_DUMP_LOCATION in desktop_browser_backend.
    # Therefore, we are adding a 5 second wait for now to see if this can help
    # the problem until we determine if there is another chrome process we can
    # wait on to ensure that all browser load processes have finished
    time.sleep(5)
    new_tab = self._browser.tabs.New()
    new_tab.Navigate(self.UrlOfUnittestFile('blank.html'),
        script_to_evaluate_on_commit=script)
    # Wait until the javascript has run ensuring that
    # the new browser has restarted before we crash it again
    py_utils.WaitFor(lambda: new_tab.EvaluateJavaScript(value), 60)