#!/usr/bin/env vpython3 # Copyright 2018 The Chromium Authors # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """ Unittest on process_perf_results.py Example usage: vpython3 process_perf_results_unittest.py """ from __future__ import absolute_import import json import os import shutil import tempfile import unittest from unittest import mock import six from core import path_util path_util.AddBuildUtilToPath() from lib.common import google_storage_helper # pylint: disable=import-error path_util.AddTelemetryToPath() from telemetry import decorators import process_perf_results as ppr_module import json_util UUID_SIZE = 36 BUILTIN_MODULE = '__builtin__' if six.PY2 else 'builtins' class _FakeLogdogStream(object): def write(self, data): del data # unused def close(self): pass def get_viewer_url(self): return 'http://foobar.not.exit' # pylint: disable=protected-access class DataFormatParsingUnitTest(unittest.TestCase): def tearDown(self): ppr_module._data_format_cache = {} def testGtest(self): with mock.patch(BUILTIN_MODULE + '.open', mock.mock_open(read_data='{}')): self.assertTrue(ppr_module._is_gtest('test.json')) self.assertFalse(ppr_module._is_histogram('test.json')) self.assertTrue(ppr_module._is_gtest('test.json')) self.assertFalse(ppr_module._is_histogram('test.json')) def testChartJSON(self): with mock.patch(BUILTIN_MODULE + '.open', mock.mock_open(read_data='{"charts": 1}')): self.assertFalse(ppr_module._is_gtest('test.json')) self.assertFalse(ppr_module._is_histogram('test.json')) self.assertFalse(ppr_module._is_gtest('test.json')) self.assertFalse(ppr_module._is_histogram('test.json')) def testHistogram(self): with mock.patch(BUILTIN_MODULE + '.open', mock.mock_open(read_data='[]')): self.assertTrue(ppr_module._is_histogram('test.json')) self.assertFalse(ppr_module._is_gtest('test.json')) self.assertTrue(ppr_module._is_histogram('test.json')) self.assertFalse(ppr_module._is_gtest('test.json')) class ProcessPerfResultsIntegrationTest(unittest.TestCase): def setUp(self): self.test_dir = tempfile.mkdtemp() self.output_json = os.path.join(self.test_dir, 'output.json') self.task_output_dir = os.path.join( os.path.dirname(__file__), 'testdata', 'task_output_dir') m1 = mock.patch( 'process_perf_results.logdog_helper.text', return_value = 'http://foo.link') m1.start() self.addCleanup(m1.stop) m2 = mock.patch( 'process_perf_results.logdog_helper.open_text', return_value=_FakeLogdogStream()) m2.start() self.addCleanup(m2.stop) m3 = mock.patch('core.results_dashboard.SendResults') m3.start() self.addCleanup(m3.stop) def tearDown(self): shutil.rmtree(self.test_dir) @decorators.Disabled('chromeos') # crbug.com/865800 @decorators.Disabled('win') # crbug.com/860677, mock doesn't integrate well # with multiprocessing on Windows. @decorators.Disabled('all') # crbug.com/967125 def testIntegration(self): build_properties = json.dumps({ 'perf_dashboard_machine_group': 'test-builder', 'buildername': 'test-builder', 'buildnumber': 777, 'got_v8_revision': 'beef1234', 'got_revision_cp': 'refs/heads/master@{#1234}', 'got_webrtc_revision': 'fee123', 'git_revision': 'deadbeef', 'buildbucket': r"""{"build": {"bucket": "master.tryserver.chromium.perf", "created_by": "user:foo", "created_ts": "1535490272757820", "id": "8936915467712010816", "project": "chrome", "lease_key": "461228535", "tags": ["builder:android-go-perf", "buildset:patch/1194825/3", "cq_experimental:False", "master:master.tryserver.chromium.perf", "user_agent:cq"]}}""" }) output_results_dir = os.path.join(self.test_dir, 'outputresults') os.mkdir(output_results_dir) return_code, benchmark_upload_result_map = ppr_module.process_perf_results( self.output_json, configuration_name='test-builder', build_properties=build_properties, task_output_dir=self.task_output_dir, smoke_test_mode=False, output_results_dir=output_results_dir) # Output filenames are prefixed with a UUID. Strip it off. output_results = { filename[UUID_SIZE:]: os.stat(os.path.join( output_results_dir, filename)).st_size for filename in os.listdir(output_results_dir)} self.assertEqual(31, len(output_results)) self.assertLess(10 << 10, output_results['power.desktop.reference']) self.assertLess(10 << 10, output_results['blink_perf.image_decoder']) self.assertLess(10 << 10, output_results['octane.reference']) self.assertLess(10 << 10, output_results['power.desktop']) self.assertLess(10 << 10, output_results['speedometer-future']) self.assertLess(10 << 10, output_results['blink_perf.owp_storage']) self.assertLess(10 << 10, output_results['memory.desktop']) self.assertLess(10 << 10, output_results['wasm']) self.assertLess(10 << 10, output_results[ 'dummy_benchmark.histogram_benchmark_1']) self.assertLess(10 << 10, output_results[ 'dummy_benchmark.histogram_benchmark_1.reference']) self.assertLess(10 << 10, output_results['wasm.reference']) self.assertLess(10 << 10, output_results['speedometer']) self.assertLess(10 << 10, output_results[ 'memory.long_running_idle_gmail_tbmv2']) self.assertLess(10 << 10, output_results['v8.runtime_stats.top_25']) self.assertLess(1 << 10, output_results[ 'dummy_benchmark.noisy_benchmark_1']) self.assertLess(10 << 10, output_results['blink_perf.svg']) self.assertLess(10 << 10, output_results[ 'v8.runtime_stats.top_25.reference']) self.assertLess(10 << 10, output_results['jetstream.reference']) self.assertLess(10 << 10, output_results['jetstream']) self.assertLess(10 << 10, output_results['speedometer2-future.reference']) self.assertLess(10 << 10, output_results['blink_perf.svg.reference']) self.assertLess(10 << 10, output_results[ 'blink_perf.image_decoder.reference']) self.assertLess(10 << 10, output_results['power.idle_platform.reference']) self.assertLess(10 << 10, output_results['power.idle_platform']) self.assertLess(1 << 10, output_results[ 'dummy_benchmark.noisy_benchmark_1.reference']) self.assertLess(10 << 10, output_results['speedometer-future.reference']) self.assertLess(10 << 10, output_results[ 'memory.long_running_idle_gmail_tbmv2.reference']) self.assertLess(10 << 10, output_results['memory.desktop.reference']) self.assertLess(10 << 10, output_results[ 'blink_perf.owp_storage.reference']) self.assertLess(10 << 10, output_results['octane']) self.assertLess(10 << 10, output_results['speedometer.reference']) self.assertEqual(return_code, 1) self.assertEqual( benchmark_upload_result_map, { 'power.desktop.reference': True, 'blink_perf.image_decoder': True, 'octane.reference': True, 'power.desktop': True, 'speedometer-future': True, 'blink_perf.owp_storage': True, 'memory.desktop': True, 'wasm': True, 'dummy_benchmark.histogram_benchmark_1': True, 'dummy_benchmark.histogram_benchmark_1.reference': True, 'wasm.reference': True, 'speedometer': True, 'memory.long_running_idle_gmail_tbmv2': True, 'v8.runtime_stats.top_25': True, 'dummy_benchmark.noisy_benchmark_1': True, 'blink_perf.svg': True, 'v8.runtime_stats.top_25.reference': True, 'jetstream.reference': True, 'jetstream': True, 'speedometer2-future.reference': True, 'speedometer2-future': False, # Only this fails due to malformed data 'blink_perf.svg.reference': True, 'blink_perf.image_decoder.reference': True, 'power.idle_platform.reference': True, 'power.idle_platform': True, 'dummy_benchmark.noisy_benchmark_1.reference': True, 'speedometer-future.reference': True, 'memory.long_running_idle_gmail_tbmv2.reference': True, 'memory.desktop.reference': True, 'blink_perf.owp_storage.reference': True, 'octane': True, 'speedometer.reference': True }) class ProcessPerfResults_HardenedUnittest(unittest.TestCase): def setUp(self): self._logdog_text = mock.patch( 'process_perf_results.logdog_helper.text', return_value = 'http://foo.link') self._logdog_text.start() self.addCleanup(self._logdog_text.stop) self._logdog_open_text = mock.patch( 'process_perf_results.logdog_helper.open_text', return_value=_FakeLogdogStream()) self._logdog_open_text.start() self.addCleanup(self._logdog_open_text.stop) @decorators.Disabled('chromeos') # crbug.com/956178 def test_handle_perf_json_test_results_IOError(self): directory_map = { 'benchmark.example': ['directory_that_does_not_exist']} test_results_list = [] ppr_module._handle_perf_json_test_results(directory_map, test_results_list) self.assertEqual(test_results_list, []) @decorators.Disabled('chromeos') # crbug.com/956178 def test_last_shard_has_no_tests(self): benchmark_name = 'benchmark.example' temp_parent_dir = tempfile.mkdtemp(suffix='test_results_outdir') try: shard1_dir = os.path.join(temp_parent_dir, 'shard1') os.mkdir(shard1_dir) shard2_dir = os.path.join(temp_parent_dir, 'shard2') os.mkdir(shard2_dir) with open(os.path.join(shard1_dir, 'test_results.json'), 'w') as fh: fh.write( '{"version": 3, "tests":{"v8.browsing_desktop-future": "blah"}}') with open(os.path.join(shard2_dir, 'test_results.json'), 'w') as fh: fh.write('{"version": 3,"tests":{}}') directory_map = { benchmark_name: [shard1_dir, shard2_dir]} benchmark_enabled_map = ppr_module._handle_perf_json_test_results( directory_map, []) self.assertTrue(benchmark_enabled_map[benchmark_name], 'Regression test for crbug.com/984565') finally: shutil.rmtree(temp_parent_dir) @decorators.Disabled('chromeos') # crbug.com/956178 def test_merge_perf_results_IOError(self): results_filename = None directories = ['directory_that_does_not_exist'] ppr_module._merge_perf_results('benchmark.example', results_filename, directories) @decorators.Disabled('chromeos') # crbug.com/956178 def test_handle_perf_logs_no_log(self): tempdir = tempfile.mkdtemp() try: dir1 = os.path.join(tempdir, '1') dir2 = os.path.join(tempdir, '2') os.makedirs(dir1) os.makedirs(dir2) with open(os.path.join(dir1, 'benchmark_log.txt'), 'w') as logfile: logfile.write('hello world') directory_map = { 'benchmark.with.log': [dir1], 'benchmark.with.no.log': [dir2], } extra_links = {} ppr_module._handle_perf_logs(directory_map, extra_links) finally: shutil.rmtree(tempdir) class ProcessPerfResults_PerfSkiaJsonUnittest(unittest.TestCase): @mock.patch.object(json_util, 'gcs_buckets_from_builder_name', autospec=True, return_value=['chrome-perf-dashboard-test']) @mock.patch.object(json_util, 'is_empty', autospec=True, return_value=False) @mock.patch('builtins.open', new_callable=mock.mock_open) @mock.patch.object(json, 'dump', autospec=True) @mock.patch.object(ppr_module, '_process_skia_json', autospec=True) @mock.patch.object(json_util, 'get_gcs_prefix_path', autospec=True) @mock.patch.object(google_storage_helper, 'upload', autospec=True, return_value='') @mock.patch.object(google_storage_helper, 'unique_name', autospec=True) def test_upload_skia_json(self, mock_unique_name, mock_upload, mock_get_gcs_prefix_path, mock_process_skia_json, mock_dump, mock_file_open, mock_is_empty, _): key = { 'improvement_direction': 'down', 'unit': 'ms_smallerIsBetter', 'test': 'Editor-TipTap', 'subtest_1': 'Speedometer3', } measurement = { 'measurements': { 'stat': [ { 'value': 'value', 'measurement': 140.6900000002235 }, { 'value': 'error', 'measurement': 13.676537086499565 }, { 'value': 'count', 'measurement': 10.0 }, { 'value': 'max', 'measurement': 172.90000000130385 }, { 'value': 'min', 'measurement': 130.90000000037253 }, { 'value': 'sum', 'measurement': 1406.9000000022352 }, ] }, 'key': key, } links = { 'Build Page': ('https://ci.chromium.org/ui/p/chrome/builders/ci/win-10-perf/39376'), 'OS Version': '10.0.19045', 'Bot Id': 'win-222-e504', 'Chromium Commit Position': 'https://crrev.com/1405221', 'V8 Git Hash': ('https://chromium.googlesource.com/v8/v8/+/' '60e67b93909a1c858305b27111d9988f94fff0f8'), 'WebRTC Git Hash': ('https://webrtc.googlesource.com/src/+/' '1e19045eaa63d00a3b4017fd43c5b502c6ed73a2'), } skia_json = { 'version': 1, 'git_hash': 'CP:1405221', 'key': { 'master': 'ChromiumPerf', 'bot': 'win-10-perf', 'benchmark': 'speedometer3', }, 'results': [measurement], 'links': links, } mock_process_skia_json.return_value = skia_json mock_get_gcs_prefix_path.return_value = ( 'ingest/2024/08/29/ChromiumPerf' '/win-11-perf/9719/benchmark.example/skia_results.json') mock_unique_name.return_value = ('skia_results_benchmark.example_' 'win-11-perf_9719_2024_08_25_T00_39_41-UTC' '.json') benchmark_name = 'benchmark.example' configuration_name = 'win-11-perf' results_filename = 'results.json' tmpfile_dir = 'tmpfile_dir' build_properties = { 'buildername': 'win-11-perf', 'buildnumber': 9719, 'got_revision_cp': 'refs/heads/main@{#1415171}', 'got_v8_revision': '0f87a54dade4353b6ece1d7591ca8c66f90c1c93', 'got_webrtc_revision': '0533b5eafe69b744f10fa178f5a6f9657eaeeb25', 'perf_dashboard_machine_group': 'ChromiumPerf', } builder_details = json_util.PerfBuilderDetails( bot='win-11-perf', builder_page=('https://ci.chromium.org/ui/p/chrome/builders/ci/' 'win-11-perf/9719'), git_hash='CP:1415171', master='ChromiumPerf', chromium_commit_position='refs/heads/main@{#1415171}', v8_git_hash='0f87a54dade4353b6ece1d7591ca8c66f90c1c93', webrtc_git_hash='0533b5eafe69b744f10fa178f5a6f9657eaeeb25', ) logdog_benchmark_dict = { 'skia_json_conversion_failed': 'False', 'skia_json_upload_failed': 'False', } got = ppr_module._upload_skia_json( benchmark_name=benchmark_name, configuration_name=configuration_name, results_filename=results_filename, tmpfile_dir=tmpfile_dir, build_properties=build_properties, logdog_benchmark_dict=logdog_benchmark_dict) self.assertEqual(got, 0) mock_process_skia_json.assert_called_once_with( results_filename=results_filename, builder_details=builder_details, benchmark_name=benchmark_name) mock_get_gcs_prefix_path.assert_called_once_with( build_properties=build_properties, builder_details=builder_details, benchmark_name=benchmark_name, given_datetime=None, filename=('skia_results_benchmark.example_' 'win-11-perf_9719_2024_08_25_T00_39_41-UTC.json')) skia_results_filepath = os.path.join('tmpfile_dir', 'skia_results.json') mock_upload.assert_called_once_with( name=('ingest/2024/08/29/ChromiumPerf/win-11-perf/9719/' 'benchmark.example/skia_results.json'), filepath=skia_results_filepath, bucket='chrome-perf-dashboard-test', content_type='application/json', authenticated_link=True) mock_dump.assert_called_once_with(skia_json, mock.ANY) mock_file_open.assert_called_once_with(skia_results_filepath, 'w') mock_is_empty.assert_called_once_with(skia_json) if __name__ == '__main__': unittest.main()