1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
|
# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import argparse
def ParseArgs():
parser = argparse.ArgumentParser(
description=('Script for automatically suppressing flaky/failing '
'tests.'))
parser.add_argument('--project',
required=True,
help=('The billing project to use for BigQuery queries. '
'Must have access to the ResultDB BQ tables, e.g. '
'"chrome-luci-data.chromium.gpu_ci_test_results".'))
parser.add_argument('--sample-period',
type=int,
default=7,
choices=range(1, 30),
help=('The number of days to sample data from.'))
parser.add_argument('--no-group-by-tags',
action='store_false',
default=True,
dest='group_by_tags',
help=('Append added expectations to the end of the file '
'instead of attempting to automatically group with '
'similar expectations.'))
parser.add_argument('--no-prompt-for-user-input',
action='store_false',
default=True,
dest='prompt_for_user_input',
help=('Generate expectations automatically based on '
'thresholds instead of prompting the user each '
'time. The user will still need to add associated '
'bugs to generated expectations afterwards.'))
parser.add_argument('--ignore-threshold',
type=float,
default=0.01,
help=('The fraction of failed tests under which flakes '
'will be ignored instead of having an expectation '
'added when --no-prompt-for-user-input is used.'))
parser.add_argument('--flaky-threshold',
type=float,
default=0.5,
help=('The fraction of failed tests under which flakes '
'will be marked as RetryOnFailure when '
'--no-prompt-for-user-input is used. Above this, '
'failures will be marked as Failure.'))
parser.add_argument('--include-all-tags',
action='store_true',
default=False,
help=('Use all tags generated by a configuration when '
'creating an expectation rather than attempting '
'to only use the most specific one. This should '
'only need to be passed if the tags in the '
'expectation files are not ordered from least '
'specific to most specific.'))
parser.add_argument('--result-output-file',
help=('Output file to store the generated results. If '
'not specified, will use a temporary file.'))
parser.add_argument('--bypass-up-to-date-check',
action='store_true',
default=False,
help=('Bypasses the check that the local expectation '
'files are up to date. Only intended for use on '
'bots to avoid failures due to potential race '
'conditions between updating the checkout and '
'running the script.'))
parser.add_argument(
'--non-hidden-failures-only',
action='store_true',
default=False,
help=
('Enable this option to only targeting visible failures on CI builders. '
'The test results will fail the builder runs, flaky results will '
'consider as pass in this option.'))
parser.add_argument(
'--build-fail-total-number-threshold',
type=int,
default=10,
help=('Threshold based on failed build number when '
'--non-hidden-failures-only is used. A test will be '
'suppressed if its failed build number is equal to or more than '
'this threshold. All --build-fail*-thresholds must be hit in '
'order for a test to actually be suppressed.'))
parser.add_argument(
'--build-fail-consecutive-days-threshold',
type=int,
default=2,
choices=range(1, 30),
help=('Threshold based on number of consecutive days that non-hidden'
'failures occur. A test will be suppressed if the number of'
'consecutive days that it has non-hidden failures is equal'
'to or more than this threshold. All --build-fail*-thresholds '
'must be hit in order for a test to actually be suppressed.'))
parser.add_argument('--build-fail-recent-days-threshold',
type=int,
default=2,
choices=range(1, 30),
help=('Suppress tests with non-hidden build failures'
' within |build_fail_recent_day_threshold| days '
'when all other build-fail* thresholds meet.'))
parser.add_argument('--builder-name',
default=[],
action='append',
dest='builder_names',
help='CI builder list to suppress tests.')
args = parser.parse_args()
if not args.prompt_for_user_input:
if args.ignore_threshold < 0:
raise ValueError('--ignore-threshold must be positive')
if args.flaky_threshold < 0:
raise ValueError('--flaky-threshold must be positive')
if args.flaky_threshold <= args.ignore_threshold:
raise ValueError(
'--flaky-threshold must be greater than --ignore-threshold')
if args.build_fail_total_number_threshold < 0:
raise ValueError('--build-fail-total-number-threshold must be positive')
return args
|
