# This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. import contextlib import os import pathlib import shutil import tempfile from unittest import mock import mozunit import pytest from tryselect.selectors.perf import ( MAX_PERF_TASKS, Apps, InvalidCategoryException, InvalidRegressionDetectorQuery, PerfParser, Platforms, Suites, Variants, run, ) from tryselect.selectors.perf_preview import plain_display from tryselect.selectors.perfselector.classification import ( check_for_live_sites, check_for_profile, ) from tryselect.selectors.perfselector.perfpushinfo import PerfPushInfo here = os.path.abspath(os.path.dirname(__file__)) FTG_SAMPLE_PATH = pathlib.Path(here, "full-task-graph-perf-test.json") TASKS = [ "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-animometer-1-3", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-optimizing", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-webaudio", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-speedometer", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-jetstream2", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-ares6", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-optimizing", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-sunspider", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-matrix-react-bench", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot-baseline", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-twitch-animation", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-assorted-dom", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-stylebench", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-misc-baseline", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-motionmark-htmlsuite-1-3", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-firefox-unity-webgl", "test-linux1804-64-shippable-qr/opt-browsertime-benchmark-wasm-firefox-wasm-godot", "test-linux1804-64-shippable-qr/opt-browsertime-amazon", ] # The TEST_VARIANTS, and TEST_CATEGORIES are used to force # a particular set of categories to show up in testing. Otherwise, # every time someone adds a category, or a variant, we'll need # to redo all the category counts. The platforms, and apps are # not forced because they change infrequently. TEST_VARIANTS = { # Bug 1837058 - Switch this back to Variants.NO_FISSION when # the default flips to fission on android Variants.FISSION.value: { "query": "'nofis", "negation": "!nofis", "platforms": [Platforms.ANDROID.value], "apps": [Apps.FENIX.value, Apps.GECKOVIEW.value], }, Variants.BYTECODE_CACHED.value: { "query": "'bytecode", "negation": "!bytecode", "platforms": [Platforms.DESKTOP.value], "apps": [Apps.FIREFOX.value], }, Variants.LIVE_SITES.value: { "query": "'live", "negation": "!live", "restriction": check_for_live_sites, "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], "apps": list(PerfParser.apps.keys()), }, Variants.PROFILING.value: { "query": "'profil", "negation": "!profil", "restriction": check_for_profile, "platforms": [Platforms.DESKTOP.value, Platforms.ANDROID.value], "apps": [Apps.FIREFOX.value, Apps.GECKOVIEW.value, Apps.FENIX.value], }, Variants.SWR.value: { "query": "'swr", "negation": "!swr", "platforms": [Platforms.DESKTOP.value], "apps": [Apps.FIREFOX.value], }, } TEST_CATEGORIES = { "Pageload": { "query": { Suites.RAPTOR.value: ["'browsertime 'tp6"], }, "suites": [Suites.RAPTOR.value], "tasks": [], "description": "", }, "Pageload (essential)": { "query": { Suites.RAPTOR.value: ["'browsertime 'tp6 'essential"], }, "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, "suites": [Suites.RAPTOR.value], "tasks": [], "description": "", }, "Responsiveness": { "query": { Suites.RAPTOR.value: ["'browsertime 'responsive"], }, "suites": [Suites.RAPTOR.value], "variant-restrictions": {Suites.RAPTOR.value: []}, "tasks": [], "description": "", }, "Benchmarks": { "query": { Suites.RAPTOR.value: ["'browsertime 'benchmark"], }, "suites": [Suites.RAPTOR.value], "variant-restrictions": {Suites.RAPTOR.value: []}, "tasks": [], "description": "", }, "DAMP (Devtools)": { "query": { Suites.TALOS.value: ["'talos 'damp"], }, "suites": [Suites.TALOS.value], "tasks": [], "description": "", }, "Talos PerfTests": { "query": { Suites.TALOS.value: ["'talos"], }, "suites": [Suites.TALOS.value], "tasks": [], "description": "", }, "Resource Usage": { "query": { Suites.TALOS.value: ["'talos 'xperf | 'tp5"], Suites.RAPTOR.value: ["'power 'osx"], Suites.AWSY.value: ["'awsy"], }, "suites": [Suites.TALOS.value, Suites.RAPTOR.value, Suites.AWSY.value], "platform-restrictions": [Platforms.DESKTOP.value], "variant-restrictions": { Suites.RAPTOR.value: [], Suites.TALOS.value: [], }, "app-restrictions": { Suites.RAPTOR.value: [Apps.FIREFOX.value], Suites.TALOS.value: [Apps.FIREFOX.value], }, "tasks": [], "description": "", }, "Graphics, & Media Playback": { "query": { # XXX This might not be an exhaustive list for talos atm Suites.TALOS.value: ["'talos 'svgr | 'bcv | 'webgl"], Suites.RAPTOR.value: ["'browsertime 'youtube-playback"], }, "suites": [Suites.TALOS.value, Suites.RAPTOR.value], "variant-restrictions": {Suites.RAPTOR.value: [Variants.FISSION.value]}, "tasks": [], "description": "", }, "Machine Learning": { "query": { Suites.PERFTEST.value: ["'perftest '-ml-"], }, "suites": [Suites.PERFTEST.value], "platform-restrictions": [ Platforms.DESKTOP.value, Platforms.LINUX.value, Platforms.MACOSX.value, Platforms.WINDOWS.value, ], "app-restrictions": { Suites.PERFTEST.value: [ Apps.FIREFOX.value, ], }, "tasks": [], "description": ( "A set of tests used to test machine learning performance in Firefox." ), }, } @contextlib.contextmanager def category_reset(): try: original_categories = PerfParser.categories yield finally: PerfParser.categories = original_categories def setup_perfparser(): PerfParser.categories = TEST_CATEGORIES PerfParser.variants = TEST_VARIANTS PerfParser.push_info = PerfPushInfo() @pytest.mark.parametrize( "category_options, expected_counts, unique_categories, missing", [ # Default should show the premade live category, but no chrome or android # The benchmark desktop category should be visible in all configurations # except for when there are requested apps/variants/platforms ( {}, 96, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Pageload macosx": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Resource Usage desktop": { "awsy": ["'awsy", "!android 'shippable !-32 !clang"], "raptor": [ "'power 'osx", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ], "talos": [ "'talos 'xperf | 'tp5", "!android 'shippable !-32 !clang", "!profil", "!swr", ], }, "Machine Learning desktop firefox": { "perftest": [ "'perftest '-ml-", "!android", "!chrom !geckoview !fenix !safari !m-car !safari-tp", ], }, }, [ "Responsiveness android-p2 geckoview", ], ), # Default settings ( {"live_sites": True}, 110, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Pageload macosx": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "!bytecode", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Pageload macosx live-sites": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "'live", "!bytecode", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ], }, }, [ "Responsiveness android-p2 geckoview", "Benchmarks desktop firefox profiling", "Talos desktop live-sites", "Talos desktop profiling+swr", "Benchmarks desktop firefox live-sites+profiling" "Benchmarks desktop firefox live-sites", ], ), ( {"live_sites": True, "safari": True}, 116, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!profil", "!chrom", "!m-car", "!safari-tp", ] }, "Pageload macosx safari": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "'safari", "!bytecode", "!profil", ] }, "Pageload macosx safari live-sites": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "'safari", "'live", "!bytecode", "!profil", ], }, }, [ "Pageload linux safari", "Pageload desktop safari", ], ), ( {"safari-tp": True}, 96, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Pageload macosx": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, }, [ "Pageload linux safari-tp", "Pageload windows safari-tp", "Pageload desktop safari-tp", ], ), ( {"live_sites": True, "chrome": True}, 146, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!profil", "!safari", "!m-car", "!safari-tp", ] }, "Pageload macosx live-sites": { "raptor": [ "'browsertime 'tp6", "'osx 'shippable", "'live", "!bytecode", "!profil", "!safari", "!m-car", "!safari-tp", ], }, }, [ "Responsiveness android-p2 geckoview", "Firefox Pageload linux chrome", "Talos PerfTests desktop swr", ], ), ( {"android": True}, 96, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ], }, "Responsiveness android-a55 geckoview": { "raptor": [ "'browsertime 'responsive", "'android 'a55 'shippable 'aarch64", "'geckoview", "!nofis", "!live", "!profil", ], }, }, [ "Responsiveness android-a55 chrome-m", "Firefox Pageload android", ], ), ( {"android": True, "chrome": True}, 126, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!profil", "!safari", "!m-car", "!safari-tp", ], }, "Responsiveness android-a55 chrome-m": { "raptor": [ "'browsertime 'responsive", "'android 'a55 'shippable 'aarch64", "'chrome-m", "!nofis", "!live", "!profil", ], }, }, ["Responsiveness android-p2 chrome-m", "Resource Usage android"], ), ( {"android": True, "chrome": True, "profile": True}, 164, { "Benchmarks desktop": { "raptor": [ "'browsertime 'benchmark", "!android 'shippable !-32 !clang", "!bytecode", "!live", "!safari", "!m-car", "!safari-tp", ] }, "Talos PerfTests desktop profiling": { "talos": [ "'talos", "!android 'shippable !-32 !clang", "'profil", "!swr", ] }, }, [ "Resource Usage desktop profiling", "DAMP (Devtools) desktop chrome", "Resource Usage android", ], ), ( {"android": True, "fenix": True}, 96, { "Pageload android-a55": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "!nofis", "!live", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ] }, "Pageload android-a55 fenix": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "!nofis", "!live", "!profil", ] }, }, [ "Resource Usage desktop profiling", "DAMP (Devtools) desktop chrome", "Resource Usage android", ], ), # Show all available windows tests, no other platform should exist # including the desktop catgeory ( {"requested_platforms": ["windows"]}, 16, { "Benchmarks windows firefox": { "raptor": [ "'browsertime 'benchmark", "!-32 !10-64 'windows 'shippable", "!chrom !geckoview !fenix !safari !m-car !safari-tp", "!bytecode", "!live", "!profil", ] }, }, [ "Resource Usage desktop", "Benchmarks desktop", "Benchmarks linux firefox bytecode-cached+profiling", ], ), # Can't have fenix on the windows platform ( {"requested_platforms": ["windows"], "requested_apps": ["fenix"]}, 0, {}, ["Benchmarks desktop"], ), # There should be no global categories available, only fenix ( { "requested_platforms": ["android"], "requested_apps": ["fenix"], "android": True, }, 10, { "Pageload android fenix": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "!nofis", "!live", "!profil", ], } }, ["Benchmarks desktop", "Pageload (live) android"], ), # Test with multiple apps ( { "requested_platforms": ["android"], "requested_apps": ["fenix", "geckoview"], "android": True, }, 15, { "Benchmarks android geckoview": { "raptor": [ "'browsertime 'benchmark", "'android 'a55 'shippable 'aarch64", "'geckoview", "!nofis", "!live", "!profil", ], }, "Pageload android fenix": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "!nofis", "!live", "!profil", ], }, }, [ "Benchmarks desktop", "Pageload android no-fission", "Pageload android fenix live-sites", ], ), # Variants are inclusive, so we'll see the variant alongside the # base here for fenix ( { "requested_variants": ["fission"], "requested_apps": ["fenix"], "android": True, }, 32, { "Pageload android-a55 fenix": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "!live", "!profil", ], }, "Pageload android-a55 fenix fission": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "'nofis", "!live", "!profil", ], }, "Pageload (essential) android fenix fission": { "raptor": [ "'browsertime 'tp6 'essential", "'android 'a55 'shippable 'aarch64", "'fenix", "'nofis", "!live", "!profil", ], }, }, [ "Benchmarks desktop", "Pageload (live) android", "Pageload android-p2 fenix live-sites", ], ), # With multiple variants, we'll see the base variant (with no combinations) # for each of them ( { "requested_variants": ["fission", "live-sites"], "requested_apps": ["fenix"], "android": True, }, 40, { "Pageload android-a55 fenix": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "!profil", ], }, "Pageload android-a55 fenix fission": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "'nofis", "!live", "!profil", ], }, "Pageload android-a55 fenix live-sites": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "'live", "!nofis", "!profil", ], }, "Pageload (essential) android fenix fission": { "raptor": [ "'browsertime 'tp6 'essential", "'android 'a55 'shippable 'aarch64", "'fenix", "'nofis", "!live", "!profil", ], }, "Pageload android fenix fission+live-sites": { "raptor": [ "'browsertime 'tp6", "'android 'a55 'shippable 'aarch64", "'fenix", "'nofis", "'live", "!profil", ], }, }, [ "Benchmarks desktop", "Pageload (live) android", "Pageload android-p2 fenix live-sites", "Pageload (essential) android fenix no-fission+live-sites", ], ), # Make sure that no no-fission tasks are selected when a variant cannot # run on a requested platform ( { "requested_variants": ["no-fission"], "requested_platforms": ["windows"], }, 16, { "Responsiveness windows firefox": { "raptor": [ "'browsertime 'responsive", "!-32 !10-64 'windows 'shippable", "!chrom !geckoview !fenix !safari !m-car !safari-tp", "!bytecode", "!live", "!profil", ], }, }, ["Benchmarks desktop", "Responsiveness windows firefox no-fisson"], ), # We should only see the base and the live-site variants here for windows ( { "requested_variants": ["no-fission", "live-sites"], "requested_platforms": ["windows"], "android": True, }, 18, { "Responsiveness windows firefox": { "raptor": [ "'browsertime 'responsive", "!-32 !10-64 'windows 'shippable", "!chrom !geckoview !fenix !safari !m-car !safari-tp", "!bytecode", "!profil", ], }, "Pageload windows live-sites": { "raptor": [ "'browsertime 'tp6", "!-32 !10-64 'windows 'shippable", "'live", "!bytecode", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ], }, "Graphics, & Media Playback windows": { "raptor": [ "'browsertime 'youtube-playback", "!-32 !10-64 'windows 'shippable", "!bytecode", "!profil", "!chrom", "!safari", "!m-car", "!safari-tp", ], "talos": [ "'talos 'svgr | 'bcv | 'webgl", "!-32 !10-64 'windows 'shippable", "!profil", "!swr", ], }, "Machine Learning windows": { "perftest": [ "'perftest '-ml-", "'windows", "!chrom", "!safari", "!m-car", "!safari-tp", ], }, }, [ "Benchmarks desktop", "Responsiveness windows firefox no-fisson", "Pageload (live) android", "Talos desktop live-sites", "Talos android", "Graphics, & Media Playback windows live-sites", "Graphics, & Media Playback android no-fission", ], ), ], ) def test_category_expansion( category_options, expected_counts, unique_categories, missing ): # Set the categories, and variants to expand PerfParser.categories = TEST_CATEGORIES PerfParser.variants = TEST_VARIANTS # Expand the categories, then either check if the unique_categories, # exist or are missing from the categories expanded_cats = PerfParser.get_categories(**category_options) assert len(expanded_cats) == expected_counts assert not any([expanded_cats.get(ucat, None) is not None for ucat in missing]) assert all( [expanded_cats.get(ucat, None) is not None for ucat in unique_categories.keys()] ) # Ensure that the queries are as expected for cat_name, cat_query in unique_categories.items(): # Don't use get here because these fields should always exist assert cat_query == expanded_cats[cat_name]["queries"] @pytest.mark.parametrize( "category_options, call_counts", [ ( {}, 0, ), ( {"non_pgo": True}, 88, ), ], ) def test_category_expansion_with_non_pgo_flag(category_options, call_counts): PerfParser.categories = TEST_CATEGORIES PerfParser.variants = TEST_VARIANTS expanded_cats = PerfParser.get_categories(**category_options) non_shippable_count = 0 for cat_name in expanded_cats: queries = str(expanded_cats[cat_name].get("queries", None)) if "!shippable !nightlyasrelease" in queries and "'shippable" not in queries: non_shippable_count += 1 assert non_shippable_count == call_counts @pytest.mark.parametrize( "options, call_counts, log_ind, expected_log_message", [ ( {}, [10, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=13\n" ), ), ( {"query": "'Pageload 'linux 'firefox"}, [10, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=13\n" ), ), ( {"cached_revision": "cached_base_revision"}, [10, 1, 1, 10, 2, 0], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=cached_base_revision&newRev=revision&" "baseRepo=try&newRepo=try&framework=13\n" ), ), ( {"dry_run": True}, [10, 1, 1, 4, 2, 0], 2, ( "If you need any help, you can find us in the #perf-help Matrix channel:\n" "https://matrix.to/#/#perf-help:mozilla.org\n" ), ), ( {"show_all": True}, [1, 2, 2, 8, 2, 1], 0, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=1\n" ), ), ( {"show_all": True, "query": "'shippable !32 speedometer 'firefox"}, [1, 2, 2, 8, 2, 1], 0, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=1\n" ), ), ( {"single_run": True}, [10, 1, 1, 4, 2, 0], 2, ( "If you need any help, you can find us in the #perf-help Matrix channel:\n" "https://matrix.to/#/#perf-help:mozilla.org\n" ), ), ( {"detect_changes": True}, [11, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=13\n" ), ), ( {"tests": ["amazon"]}, [7, 2, 2, 10, 2, 1], 2, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=13\n" ), ), ( {"tests": ["amazon"], "alert": "000"}, [0, 2, 2, 9, 2, 1], 1, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=1\n" ), ), ( {"tests": ["amazon"], "show_all": True}, [1, 2, 2, 8, 2, 1], 0, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-results?" "baseRev=revision&newRev=revision&baseRepo=try&newRepo=try&framework=1\n" ), ), ], ) @pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") def test_full_run(options, call_counts, log_ind, expected_log_message): with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( "tryselect.selectors.perf.run_fzf" ) as fzf, mock.patch( "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() ), mock.patch( "tryselect.selectors.perf.LogProcessor.revision", new_callable=mock.PropertyMock, return_value="revision", ) as logger, mock.patch( "tryselect.selectors.perf.PerfParser.check_cached_revision", ) as ccr, mock.patch( "tryselect.selectors.perf.PerfParser.save_revision_treeherder" ) as srt, mock.patch( "tryselect.selectors.perf.print", ) as perf_print, mock.patch( "tryselect.selectors.perf.PerfParser.set_categories_for_test" ) as tests_mock, mock.patch( "tryselect.selectors.perf.requests" ) as requests_mock: def test_mock_func(*args, **kwargs): """Used for testing any --test functionality.""" PerfParser.categories = { "test 1": { "query": {"raptor": ["test 1"]}, "suites": ["raptor"], "tasks": [], "description": "", }, "test 2": { "query": {"raptor": ["test 2"]}, "suites": ["raptor"], "tasks": [], "description": "", }, "amazon": { "query": {"raptor": ["amazon"]}, "suites": ["raptor"], "tasks": [], "description": "", }, } fzf.side_effect = [ ["", ["test 1 windows firefox"]], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ] return ["task 1", "task 2", "amazon"] tests_mock.side_effect = test_mock_func get_mock = mock.MagicMock() get_mock.status_code.return_value = 200 get_mock.json.return_value = {"tasks": ["task 1", "task 2"]} requests_mock.get.return_value = get_mock fzf_side_effects = [ ["", ["Benchmarks linux"]], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", ["Perftest Change Detector"]], ] # Number of side effects for fzf should always be greater than # or equal to the number of calls expected assert len(fzf_side_effects) >= call_counts[0] fzf.side_effect = fzf_side_effects ccr.return_value = options.get("cached_revision", None) options["push_to_vcs"] = True with category_reset(): run(**options) assert fzf.call_count == call_counts[0] assert ptt.call_count == call_counts[1] assert logger.call_count == call_counts[2] assert perf_print.call_count == call_counts[3] assert ccr.call_count == call_counts[4] assert srt.call_count == call_counts[5] assert perf_print.call_args_list[log_ind][0][0] == expected_log_message @pytest.mark.parametrize( "options, call_counts, log_ind, expected_log_message", [ ( {"tests": ["amazon"], "show_all": True}, [1, 2, 0, 8, 2, 1], 0, ( "\n!!!NOTE!!!\n You'll be able to find a performance comparison " "here once the tests are complete (the autodetected framework " "selection may not show all of your tests):\n" " https://perf.compare/compare-lando-results?" "baseLando=13&newLando=14&" "baseRepo=try&newRepo=try&framework=1\n" ), ), ], ) @pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") def test_full_run_lando(options, call_counts, log_ind, expected_log_message): with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( "tryselect.selectors.perf.run_fzf" ) as fzf, mock.patch( "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() ), mock.patch( "tryselect.selectors.perf.LogProcessor.revision", new_callable=mock.PropertyMock, return_value="revision", ) as logger, mock.patch( "tryselect.selectors.perf.PerfParser.check_cached_revision", ) as ccr, mock.patch( "tryselect.selectors.perf.PerfParser.save_revision_treeherder" ) as srt, mock.patch( "tryselect.selectors.perf.print", ) as perf_print, mock.patch( "tryselect.selectors.perf.PerfParser.set_categories_for_test" ) as tests_mock, mock.patch( "tryselect.selectors.perf.requests" ) as requests_mock: def test_mock_func(*args, **kwargs): """Used for testing any --test functionality.""" PerfParser.categories = { "test 1": { "query": {"raptor": ["test 1"]}, "suites": ["raptor"], "tasks": [], "description": "", }, "test 2": { "query": {"raptor": ["test 2"]}, "suites": ["raptor"], "tasks": [], "description": "", }, "amazon": { "query": {"raptor": ["amazon"]}, "suites": ["raptor"], "tasks": [], "description": "", }, } fzf.side_effect = [ ["", ["test 1 windows firefox"]], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ] return ["task 1", "task 2", "amazon"] tests_mock.side_effect = test_mock_func get_mock = mock.MagicMock() get_mock.status_code.return_value = 200 get_mock.json.return_value = {"tasks": ["task 1", "task 2"]} requests_mock.get.return_value = get_mock ptt.side_effect = ["13", "14"] fzf_side_effects = [ ["", ["Benchmarks linux"]], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", ["Perftest Change Detector"]], ] # Number of side effects for fzf should always be greater than # or equal to the number of calls expected assert len(fzf_side_effects) >= call_counts[0] fzf.side_effect = fzf_side_effects ccr.return_value = options.get("cached_revision", None) options["push_to_vcs"] = False with category_reset(): run(**options) assert fzf.call_count == call_counts[0] assert ptt.call_count == call_counts[1] assert logger.call_count == call_counts[2] assert perf_print.call_count == call_counts[3] assert ccr.call_count == call_counts[4] assert srt.call_count == call_counts[5] assert perf_print.call_args_list[log_ind][0][0] == expected_log_message @pytest.mark.parametrize( "options, call_counts, log_ind, expected_log_message, expected_failure", [ ( {"detect_changes": True}, [11, 0, 0, 2, 1], 1, ( "Executing raptor queries: 'browsertime 'benchmark, !clang 'linux " "'shippable, !bytecode, !live, !profil, !chrom, !safari, !m-car, !safari-tp" ), InvalidRegressionDetectorQuery, ), ], ) @pytest.mark.skipif(os.name == "nt", reason="fzf not installed on host") def test_change_detection_task_injection_failure( options, call_counts, log_ind, expected_log_message, expected_failure, ): setup_perfparser() with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( "tryselect.selectors.perf.run_fzf" ) as fzf, mock.patch( "tryselect.selectors.perf.get_repository_object", new=mock.MagicMock() ), mock.patch( "tryselect.selectors.perf.LogProcessor.revision", new_callable=mock.PropertyMock, return_value="revision", ) as logger, mock.patch( "tryselect.selectors.perf.PerfParser.check_cached_revision" ) as ccr, mock.patch( "tryselect.selectors.perf.print", ) as perf_print: fzf_side_effects = [ ["", ["Benchmarks linux"]], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ["", TASKS], ] assert len(fzf_side_effects) >= call_counts[0] fzf.side_effect = fzf_side_effects with pytest.raises(expected_failure): run(**options) assert fzf.call_count == call_counts[0] assert ptt.call_count == call_counts[1] assert logger.call_count == call_counts[2] assert perf_print.call_count == call_counts[3] assert ccr.call_count == call_counts[4] assert perf_print.call_args_list[log_ind][0][0] == expected_log_message @pytest.mark.parametrize( "query, should_fail", [ ( { "query": { # Raptor has all variants available so it # should fail on this category "raptor": ["browsertime 'live 'no-fission"], } }, True, ), ( { "query": { # Awsy has no variants defined so it shouldn't fail # on a query like this "awsy": ["browsertime 'live 'no-fission"], } }, False, ), ], ) def test_category_rules(query, should_fail): # Set the categories, and variants to expand PerfParser.categories = {"test-live": query} PerfParser.variants = TEST_VARIANTS if should_fail: with pytest.raises(InvalidCategoryException): PerfParser.run_category_checks() else: assert PerfParser.run_category_checks() # Reset the categories, and variants to expand PerfParser.categories = TEST_CATEGORIES PerfParser.variants = TEST_VARIANTS @pytest.mark.parametrize( "apk_name, apk_content, should_fail, failure_message", [ ( "real-file", "file-content", False, None, ), ("bad-file", None, True, "Path does not exist:"), ], ) def test_apk_upload(apk_name, apk_content, should_fail, failure_message): with mock.patch("tryselect.selectors.perf.subprocess") as _, mock.patch( "tryselect.selectors.perf.shutil" ) as _: temp_dir = None try: temp_dir = tempfile.mkdtemp() sample_apk = pathlib.Path(temp_dir, apk_name) if apk_content is not None: with sample_apk.open("w") as f: f.write(apk_content) if should_fail: with pytest.raises(Exception) as exc_info: PerfParser.setup_apk_upload("browsertime", str(sample_apk)) assert failure_message in str(exc_info) else: PerfParser.setup_apk_upload("browsertime", str(sample_apk)) finally: if temp_dir is not None: shutil.rmtree(temp_dir) @pytest.mark.parametrize( "args, load_data, return_value, call_counts, exists_cache_file", [ ( ( [], "base_commit", ), { "base_commit": [ { "base_revision_treeherder": "2b04563b5", "date": "2023-03-31", "tasks": [], }, ], }, "2b04563b5", [1, 0], True, ), ( ( ["task-a"], "subset_base_commit", ), { "subset_base_commit": [ { "base_revision_treeherder": "2b04563b5", "date": "2023-03-31", "tasks": ["task-a", "task-b"], }, ], }, "2b04563b5", [1, 0], True, ), ( ([], "not_exist_cached_base_commit"), { "base_commit": [ { "base_revision_treeherder": "2b04563b5", "date": "2023-03-31", "tasks": [], }, ], }, None, [1, 0], True, ), ( ( ["task-a", "task-b"], "superset_base_commit", ), { "superset_base_commit": [ { "base_revision_treeherder": "2b04563b5", "date": "2023-03-31", "tasks": ["task-a"], }, ], }, None, [1, 0], True, ), ( ([], None), {}, None, [1, 1], True, ), ( ([], None), {}, None, [0, 0], False, ), ], ) def test_check_cached_revision( args, load_data, return_value, call_counts, exists_cache_file ): with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( "tryselect.selectors.perf.json.dump" ) as dump, mock.patch( "tryselect.selectors.perf.pathlib.Path.is_file" ) as is_file, mock.patch( "tryselect.selectors.perf.pathlib.Path.open" ): load.return_value = load_data is_file.return_value = exists_cache_file result = PerfParser.check_cached_revision(*args) assert load.call_count == call_counts[0] assert dump.call_count == call_counts[1] assert result == return_value @pytest.mark.parametrize( "args, call_counts, exists_cache_file", [ ( ["base_commit", "base_revision_treeherder"], [0, 1], False, ), ( ["base_commit", "base_revision_treeherder"], [1, 1], True, ), ], ) def test_save_revision_treeherder(args, call_counts, exists_cache_file): with mock.patch("tryselect.selectors.perf.json.load") as load, mock.patch( "tryselect.selectors.perf.json.dump" ) as dump, mock.patch( "tryselect.selectors.perf.pathlib.Path.is_file" ) as is_file, mock.patch( "tryselect.selectors.perf.pathlib.Path.open" ): is_file.return_value = exists_cache_file PerfParser.push_info.base_revision = "base_revision_treeherder" PerfParser.save_revision_treeherder(TASKS, args[0], True) assert load.call_count == call_counts[0] assert dump.call_count == call_counts[1] @pytest.mark.parametrize( "total_tasks, options, call_counts, expected_log_message, expected_failure", [ ( MAX_PERF_TASKS + 1, {}, [1, 0, 0, 1], ( "\n\n----------------------------------------------------------------------------------------------\n" f"You have selected {MAX_PERF_TASKS+1} total test runs! (selected tasks({MAX_PERF_TASKS+1}) * rebuild" f" count(1) \nThese tests won't be triggered as the current maximum for a single ./mach try " f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." "\n----------------------------------------------------------------------------------------------\n\n" ), True, ), ( MAX_PERF_TASKS, {"show_all": True}, [9, 0, 0, 8], ( "For more information on the performance tests, see our " "PerfDocs here:\nhttps://firefox-source-docs.mozilla.org/testing/perfdocs/" ), False, ), ( int((MAX_PERF_TASKS + 2) / 2), { "show_all": True, "try_config_params": {"try_task_config": {"rebuild": 2}}, }, [1, 0, 0, 1], ( "\n\n----------------------------------------------------------------------------------------------\n" f"You have selected {int((MAX_PERF_TASKS + 2) / 2) * 2} total test runs! (selected tasks(" f"{int((MAX_PERF_TASKS + 2) / 2)}) * rebuild" f" count(2) \nThese tests won't be triggered as the current maximum for a single ./mach try " f"perf run is {MAX_PERF_TASKS}. \nIf this was unexpected, please file a bug in Testing :: Performance." "\n----------------------------------------------------------------------------------------------\n\n" ), True, ), (0, {}, [1, 0, 0, 1], ("No tasks selected"), True), ], ) def test_max_perf_tasks( total_tasks, options, call_counts, expected_log_message, expected_failure, ): setup_perfparser() with mock.patch("tryselect.selectors.perf.push_to_try") as ptt, mock.patch( "tryselect.selectors.perf.print", ) as perf_print, mock.patch( "tryselect.selectors.perf.LogProcessor.revision", new_callable=mock.PropertyMock, return_value="revision", ), mock.patch( "tryselect.selectors.perf.PerfParser.perf_push_to_try", new_callable=mock.MagicMock, ) as perf_push_to_try_mock, mock.patch( "tryselect.selectors.perf.PerfParser.get_perf_tasks" ) as get_perf_tasks_mock, mock.patch( "tryselect.selectors.perf.PerfParser.get_tasks" ) as get_tasks_mock, mock.patch( "tryselect.selectors.perf.run_fzf" ) as fzf, mock.patch( "tryselect.selectors.perf.fzf_bootstrap", return_value=mock.MagicMock() ): tasks = ["a-task"] * total_tasks get_tasks_mock.return_value = tasks get_perf_tasks_mock.return_value = tasks, [], [] PerfParser.push_info.finished_run = not expected_failure options["push_to_vcs"] = True run(**options) assert perf_push_to_try_mock.call_count == 0 if expected_failure else 1 assert ptt.call_count == call_counts[1] assert perf_print.call_count == call_counts[3] assert fzf.call_count == 0 assert perf_print.call_args_list[-1][0][0] == expected_log_message @pytest.mark.parametrize( "try_config, selected_tasks, expected_try_config", [ ( {"use-artifact-builds": True, "disable-pgo": True}, ["some-android-task"], {"use-artifact-builds": False}, ), ( {"use-artifact-builds": True}, ["some-desktop-task"], {"use-artifact-builds": True}, ), ( {"use-artifact-builds": False}, ["some-android-task"], {"use-artifact-builds": False}, ), ( {"use-artifact-builds": True, "disable-pgo": True}, ["some-desktop-task", "some-android-task"], {"use-artifact-builds": False}, ), ], ) def test_artifact_mode_autodisable(try_config, selected_tasks, expected_try_config): PerfParser.setup_try_config({"try_task_config": try_config}, [], selected_tasks) assert ( try_config["use-artifact-builds"] == expected_try_config["use-artifact-builds"] ) def test_build_category_description(): base_cmd = ["--preview", '-t "{+f}"'] with mock.patch("tryselect.selectors.perf.json.dump") as dump: PerfParser.build_category_description(base_cmd, "") assert dump.call_count == 1 assert str(base_cmd).count("-d") == 1 assert str(base_cmd).count("-l") == 1 @pytest.mark.parametrize( "options, call_count", [ ({}, [1, 1, 2]), ({"show_all": True}, [0, 0, 1]), ], ) def test_preview_description(options, call_count): with mock.patch("tryselect.selectors.perf.PerfParser.perf_push_to_try"), mock.patch( "tryselect.selectors.perf.fzf_bootstrap" ), mock.patch( "tryselect.selectors.perf.PerfParser.get_perf_tasks" ) as get_perf_tasks, mock.patch( "tryselect.selectors.perf.PerfParser.get_tasks" ), mock.patch( "tryselect.selectors.perf.PerfParser.build_category_description" ) as bcd: get_perf_tasks.return_value = [], [], [] run(**options) assert bcd.call_count == call_count[0] base_cmd = ["--preview", '-t "{+f}"'] option = base_cmd[base_cmd.index("--preview") + 1].split(" ") description, line = None, None if call_count[0] == 1: PerfParser.build_category_description(base_cmd, "") option = base_cmd[base_cmd.index("--preview") + 1].split(" ") description = option[option.index("-d") + 1] line = "Current line" taskfile = option[option.index("-t") + 1] with mock.patch("tryselect.selectors.perf_preview.open"), mock.patch( "tryselect.selectors.perf_preview.pathlib.Path.open" ), mock.patch("tryselect.selectors.perf_preview.json.load") as load, mock.patch( "tryselect.selectors.perf_preview.print" ) as preview_print: load.return_value = {line: "test description"} plain_display(taskfile, description, line) assert load.call_count == call_count[1] assert preview_print.call_count == call_count[2] @pytest.mark.parametrize( "tests, tasks_found, categories_produced", [ (["amazon"], 2, 1), (["speedometer"], 1, 1), (["webaudio", "speedometer"], 3, 2), (["webaudio", "tp5n"], 3, 2), (["xperf"], 1, 1), (["awsy"], 1, 1), (["awsy", "tp5n", "amazon"], 4, 3), (["awsy", "tp5n", "xperf"], 2, 3), (["non-existent"], 0, 0), ], ) def test_test_selection(tests, tasks_found, categories_produced): with mock.patch( "tryselect.selectors.perfselector.classification.pathlib" ), mock.patch( "tryselect.selectors.perfselector.classification.json" ) as mocked_json, mock.patch( "tryselect.selectors.perfselector.classification.ScriptInfo" ): def mocked_json_load(*args, **kwargs): return { "suites": { "xperf": { "tests": ["tp5n"], } } } mocked_json.load.side_effect = mocked_json_load with category_reset(): all_tasks = PerfParser.set_categories_for_test(FTG_SAMPLE_PATH, tests) assert len(all_tasks) == tasks_found assert len(PerfParser.categories) == categories_produced @pytest.mark.parametrize( "tests, tasks_found, categories_produced", [ (["perftest_finder_ml_engine_perf.js"], 1, 1), (["perftest/test/finder/path"], 1, 1), (["perftest/test/finder/path/perftest_finder_ml_engine_perf.js"], 1, 1), (["background-resource"], 1, 1), ], ) def test_perftest_test_selection(tests, tasks_found, categories_produced): with mock.patch("pathlib.Path.is_file", return_value=True), mock.patch( "tryselect.selectors.perfselector.classification.ScriptInfo" ) as mock_script_info, mock.patch( "mozperftest.argparser.PerftestArgumentParser.parse_known_args" ) as mock_parse_args: mock_si_instance = mock_script_info.return_value mock_si_instance.get.return_value = "background-resource" mock_si_instance.script = pathlib.Path( "perftest/test/finder/path/perftest_finder_ml_engine_perf.js" ) class DummyArgs: tests = [ "testing/performance/android-resource/main-background.sh", ] mock_parse_args.return_value = (DummyArgs(), []) with category_reset(): all_tasks = PerfParser.set_categories_for_test(FTG_SAMPLE_PATH, tests) assert len(all_tasks) == tasks_found assert len(PerfParser.categories) == categories_produced def test_unknown_framework(): setup_perfparser() PerfParser.get_majority_framework(["unknown"]) assert PerfParser.push_info.framework == 1 if __name__ == "__main__": mozunit.main()