testing/flake_suppressor_common/argument_parsing.py

# Copyright 2021 The Chromium Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import argparse


def ParseArgs():
  parser = argparse.ArgumentParser(
      description=('Script for automatically suppressing flaky/failing '
                   'tests.'))
  parser.add_argument('--project',
                      required=True,
                      help=('The billing project to use for BigQuery queries. '
                            'Must have access to the ResultDB BQ tables, e.g. '
                            '"chrome-luci-data.chromium.gpu_ci_test_results".'))
  parser.add_argument('--sample-period',
                      type=int,
                      default=1,
                      help=('The number of days to sample data from.'))
  parser.add_argument('--no-group-by-tags',
                      action='store_false',
                      default=True,
                      dest='group_by_tags',
                      help=('Append added expectations to the end of the file '
                            'instead of attempting to automatically group with '
                            'similar expectations.'))
  parser.add_argument('--no-prompt-for-user-input',
                      action='store_false',
                      default=True,
                      dest='prompt_for_user_input',
                      help=('Generate expectations automatically based on '
                            'thresholds instead of prompting the user each '
                            'time. The user will still need to add associated '
                            'bugs to generated expectations afterwards.'))
  parser.add_argument('--ignore-threshold',
                      type=float,
                      default=0.01,
                      help=('The fraction of failed tests under which flakes '
                            'will be ignored instead of having an expectation '
                            'added when --no-prompt-for-user-input is used.'))
  parser.add_argument('--flaky-threshold',
                      type=float,
                      default=0.5,
                      help=('The fraction of failed tests under which flakes '
                            'will be marked as RetryOnFailure when '
                            '--no-prompt-for-user-input is used. Above this, '
                            'failures will be marked as Failure.'))
  parser.add_argument('--include-all-tags',
                      action='store_true',
                      default=False,
                      help=('Use all tags generated by a configuration when '
                            'creating an expectation rather than attempting '
                            'to only use the most specific one. This should '
                            'only need to be passed if the tags in the '
                            'expectation files are not ordered from least '
                            'specific to most specific.'))
  parser.add_argument('--result-output-file',
                      help=('Output file to store the generated results. If '
                            'not specified, will use a temporary file.'))
  parser.add_argument('--bypass-up-to-date-check',
                      action='store_true',
                      default=False,
                      help=('Bypasses the check that the local expectation '
                            'files are up to date. Only intended for use on '
                            'bots to avoid failures due to potential race '
                            'conditions between updating the checkout and '
                            'running the script.'))
  parser.add_argument(
      '--non-hidden-failures-only',
      action='store_true',
      default=False,
      help=
      ('Enable this option to only targeting visible failures on CI builders. '
       'The test results will fail the builder runs, flaky results will '
       'consider as pass in this option.'))
  parser.add_argument(
      '--build-fail-total-number-threshold',
      type=int,
      default=0,
      help=('Threshold based on failed build number when '
            '--non-hidden-failures-only is used. A test will be '
            'suppressed if its failed build number is equal to or more than '
            'this threshold. All --build-fail*-thresholds must be hit in '
            'order for a test to actually be suppressed.'))
  parser.add_argument(
      '--build-fail-consecutive-days-threshold',
      type=int,
      default=2,
      help=('Threshold based on number of consecutive days that non-hidden'
            'failures occur. A test will be suppressed if the number of'
            'consecutive days that it has non-hidden failures is equal'
            'to or more than this threshold. All --build-fail*-thresholds '
            'must be hit in order for a test to actually be suppressed.'))
  parser.add_argument('--builder-name',
                      default=[],
                      action="append",
                      dest="builder_names",
                      help="CI builder list to suppress tests.")
  args = parser.parse_args()

  if not args.prompt_for_user_input:
    if args.ignore_threshold < 0:
      raise ValueError('--ignore-threshold must be positive')
    if args.flaky_threshold < 0:
      raise ValueError('--flaky-threshold must be positive')
    if args.flaky_threshold <= args.ignore_threshold:
      raise ValueError(
          '--flaky-threshold must be greater than --ignore-threshold')
    if args.build_fail_total_number_threshold < 0:
      raise ValueError('--build-fail-total-number-threshold must be positive')
    if args.build_fail_consecutive_days_threshold < 0:
      raise ValueError('--build-fail-consecutive-days-threshold must be '
                       'positive')

  return args