1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
|
#!/usr/bin/env python3
# Copyright 2023 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# Script that compares the metrics when running Autofill captured site tests
# with some features enabled and disabled.
# Run from the root of the Chromium src directory. -h for help.
# The tool expects that the captured_sites_interactive_tests binary is built.
import argparse, json, os, urllib.parse
# Extracts the names of all non-disabled captured site tests.
def get_all_sites():
with open("chrome/test/data/autofill/captured_sites/artifacts/testcases.json",
"r") as f:
return (site["site_name"] for site in json.loads(f.read())["tests"]
if not site.get("disabled", False))
# Command line args.
parser = argparse.ArgumentParser(epilog="List of tests: " +
", ".join(get_all_sites()))
parser.add_argument(
"target",
help=("Build target of captured_sites_interactive_tests binary. "
"For example, Default. The binary should already exist."))
parser.add_argument("features", help="A comma-separated list of feature names.")
parser.add_argument(
"-r",
dest="histogram_regex",
help=("A regex matching the histogram names that should be dumped. If not "
"specified, the metrics of all histograms dumped."))
parser.add_argument(
"-o",
dest="output",
default="/tmp/",
help=("Directory to record the metrics into. Creates files per test, named"
" after the test case."))
parser.add_argument(
"-t",
dest="test",
help="Test case. If no test is specified, all tests are run.")
parser.add_argument("-s",
dest="silent",
action="store_true",
help="Don't print test output.")
args = parser.parse_args()
# The captured_sites_interactive_tests binary should be built.
captured_site_tests = "out/%s/captured_sites_interactive_tests" % args.target
assert os.path.exists(captured_site_tests)
# Runs the capture site test for `site` and scrapes the metrics.
# args.features is enabled/disabled depending on `features_enabled`.
def run_test(site, features_enabled):
cmd = "./" + captured_site_tests
cmd += (" --gtest_filter="
"All/AutofillCapturedSitesInteractiveTest.Recipe/" + site)
# Enable scraping tools. Special characters need to be escaped.
cmd += " --enable-features=AutofillCapturedSiteTestsMetricsScraper"
cmd += ":output_dir/" + urllib.parse.quote(args.output, safe="")
if args.histogram_regex is not None:
cmd += "/histogram_regex/" + urllib.parse.quote(args.histogram_regex,
safe="")
# En- or disable features.
if features_enabled:
cmd += "," + args.features
else:
cmd += " --disable-features=" + args.features
# Random arguments that the captured site tests recommend.
cmd += " --enable-pixel-output-in-tests"
cmd += " --test-launcher-timeout=10000000"
cmd += " --ui-test-action-max-timeout=10000000"
cmd += (" --vmodule=captured_sites_test_utils=2\,"
"autofill_captured_sites_interactive_uitest=1")
# Maybe disable output.
if args.silent:
cmd += " > /dev/null 2>&1"
# Run
os.system(cmd)
# Runs the captured site test `site` twice. Once with `args.features` enabled
# and once with the feature disabled.
# Diffs the metrics collected.
def run_tests_and_diff(site):
print("Testing %s..." % site)
def file_name(infix):
return "%s/%s%s.txt" % (args.output, site, infix)
# `output` is where the captured site test will write to. Rename the file
# afterwards to distinguish between enabled/disabled state.
output = file_name("")
result_enabled = file_name("_enabled")
result_disabled = file_name("_disabled")
print("Running with features enabled. Results at " + result_enabled)
run_test(site, True)
os.rename(output, result_enabled)
print("Running with features disabled. Results at " + result_disabled)
run_test(site, False)
os.rename(output, result_disabled)
print("Comparing metrics (no output means no diff)")
os.system("diff %s %s" % (result_enabled, result_disabled))
print("")
# If a test is specified, only run that specific test. Otherwise run all.
if args.test is None:
for site in get_all_sites():
run_tests_and_diff(site)
else:
run_tests_and_diff(args.test)
|
