1# Copyright 2021 The Chromium Authors 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import argparse 6 7 8def ParseArgs(): 9 parser = argparse.ArgumentParser( 10 description=('Script for automatically suppressing flaky/failing ' 11 'tests.')) 12 parser.add_argument('--project', 13 required=True, 14 help=('The billing project to use for BigQuery queries. ' 15 'Must have access to the ResultDB BQ tables, e.g. ' 16 '"chrome-luci-data.chromium.gpu_ci_test_results".')) 17 parser.add_argument('--sample-period', 18 type=int, 19 default=7, 20 choices=range(1, 30), 21 help=('The number of days to sample data from.')) 22 parser.add_argument('--no-group-by-tags', 23 action='store_false', 24 default=True, 25 dest='group_by_tags', 26 help=('Append added expectations to the end of the file ' 27 'instead of attempting to automatically group with ' 28 'similar expectations.')) 29 parser.add_argument('--no-prompt-for-user-input', 30 action='store_false', 31 default=True, 32 dest='prompt_for_user_input', 33 help=('Generate expectations automatically based on ' 34 'thresholds instead of prompting the user each ' 35 'time. The user will still need to add associated ' 36 'bugs to generated expectations afterwards.')) 37 parser.add_argument('--ignore-threshold', 38 type=float, 39 default=0.01, 40 help=('The fraction of failed tests under which flakes ' 41 'will be ignored instead of having an expectation ' 42 'added when --no-prompt-for-user-input is used.')) 43 parser.add_argument('--flaky-threshold', 44 type=float, 45 default=0.5, 46 help=('The fraction of failed tests under which flakes ' 47 'will be marked as RetryOnFailure when ' 48 '--no-prompt-for-user-input is used. Above this, ' 49 'failures will be marked as Failure.')) 50 parser.add_argument('--include-all-tags', 51 action='store_true', 52 default=False, 53 help=('Use all tags generated by a configuration when ' 54 'creating an expectation rather than attempting ' 55 'to only use the most specific one. This should ' 56 'only need to be passed if the tags in the ' 57 'expectation files are not ordered from least ' 58 'specific to most specific.')) 59 parser.add_argument('--result-output-file', 60 help=('Output file to store the generated results. If ' 61 'not specified, will use a temporary file.')) 62 parser.add_argument('--bypass-up-to-date-check', 63 action='store_true', 64 default=False, 65 help=('Bypasses the check that the local expectation ' 66 'files are up to date. Only intended for use on ' 67 'bots to avoid failures due to potential race ' 68 'conditions between updating the checkout and ' 69 'running the script.')) 70 parser.add_argument( 71 '--non-hidden-failures-only', 72 action='store_true', 73 default=False, 74 help= 75 ('Enable this option to only targeting visible failures on CI builders. ' 76 'The test results will fail the builder runs, flaky results will ' 77 'consider as pass in this option.')) 78 parser.add_argument( 79 '--build-fail-total-number-threshold', 80 type=int, 81 default=10, 82 help=('Threshold based on failed build number when ' 83 '--non-hidden-failures-only is used. A test will be ' 84 'suppressed if its failed build number is equal to or more than ' 85 'this threshold. All --build-fail*-thresholds must be hit in ' 86 'order for a test to actually be suppressed.')) 87 parser.add_argument( 88 '--build-fail-consecutive-days-threshold', 89 type=int, 90 default=2, 91 choices=range(1, 30), 92 help=('Threshold based on number of consecutive days that non-hidden' 93 'failures occur. A test will be suppressed if the number of' 94 'consecutive days that it has non-hidden failures is equal' 95 'to or more than this threshold. All --build-fail*-thresholds ' 96 'must be hit in order for a test to actually be suppressed.')) 97 parser.add_argument('--build-fail-recent-days-threshold', 98 type=int, 99 default=2, 100 choices=range(1, 30), 101 help=('Suppress tests with non-hidden build failures' 102 ' within |build_fail_recent_day_threshold| days ' 103 'when all other build-fail* thresholds meet.')) 104 parser.add_argument('--builder-name', 105 default=[], 106 action="append", 107 dest="builder_names", 108 help="CI builder list to suppress tests.") 109 args = parser.parse_args() 110 111 if not args.prompt_for_user_input: 112 if args.ignore_threshold < 0: 113 raise ValueError('--ignore-threshold must be positive') 114 if args.flaky_threshold < 0: 115 raise ValueError('--flaky-threshold must be positive') 116 if args.flaky_threshold <= args.ignore_threshold: 117 raise ValueError( 118 '--flaky-threshold must be greater than --ignore-threshold') 119 if args.build_fail_total_number_threshold < 0: 120 raise ValueError('--build-fail-total-number-threshold must be positive') 121 122 return args 123