1*6777b538SAndroid Build Coastguard Worker# Copyright 2021 The Chromium Authors 2*6777b538SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be 3*6777b538SAndroid Build Coastguard Worker# found in the LICENSE file. 4*6777b538SAndroid Build Coastguard Worker"""This is a library for working with test executables in a way that is 5*6777b538SAndroid Build Coastguard WorkerChromium-bot-friendly as specified by //docs/testing/test_executable_api.md 6*6777b538SAndroid Build Coastguard Worker 7*6777b538SAndroid Build Coastguard WorkerExample usage: 8*6777b538SAndroid Build Coastguard Worker import os 9*6777b538SAndroid Build Coastguard Worker import sys 10*6777b538SAndroid Build Coastguard Worker 11*6777b538SAndroid Build Coastguard Worker import main_program 12*6777b538SAndroid Build Coastguard Worker import rust_main_program 13*6777b538SAndroid Build Coastguard Worker 14*6777b538SAndroid Build Coastguard Worker if __name__ == '__main__': 15*6777b538SAndroid Build Coastguard Worker cmdline_parser = argparse.ArgumentParser() 16*6777b538SAndroid Build Coastguard Worker main_program.add_cmdline_args(cmdline_parser) 17*6777b538SAndroid Build Coastguard Worker ... adding other cmdline parameter definitions ... 18*6777b538SAndroid Build Coastguard Worker parsed_cmdline_args = cmdline_parser.parse_args() 19*6777b538SAndroid Build Coastguard Worker 20*6777b538SAndroid Build Coastguard Worker test_executable_wrappers = [] 21*6777b538SAndroid Build Coastguard Worker test_executable_wrappers.append( 22*6777b538SAndroid Build Coastguard Worker rust_main_program.TestExecutableWrapper(...)) 23*6777b538SAndroid Build Coastguard Worker ... 24*6777b538SAndroid Build Coastguard Worker 25*6777b538SAndroid Build Coastguard Worker main_program.main( 26*6777b538SAndroid Build Coastguard Worker test_executable_wrappers, parsed_cmdline_args, os.environ) 27*6777b538SAndroid Build Coastguard Worker""" 28*6777b538SAndroid Build Coastguard Worker 29*6777b538SAndroid Build Coastguard Workerimport argparse 30*6777b538SAndroid Build Coastguard Workerimport time 31*6777b538SAndroid Build Coastguard Worker 32*6777b538SAndroid Build Coastguard Workerimport test_filtering 33*6777b538SAndroid Build Coastguard Workerimport test_results 34*6777b538SAndroid Build Coastguard Worker 35*6777b538SAndroid Build Coastguard Worker 36*6777b538SAndroid Build Coastguard Workerdef add_cmdline_args(argparse_parser): 37*6777b538SAndroid Build Coastguard Worker """Adds test-filtering-specific cmdline parameter definitions to 38*6777b538SAndroid Build Coastguard Worker `argparse_parser`. 39*6777b538SAndroid Build Coastguard Worker 40*6777b538SAndroid Build Coastguard Worker Args: 41*6777b538SAndroid Build Coastguard Worker argparse_parser: An object of argparse.ArgumentParser type. 42*6777b538SAndroid Build Coastguard Worker """ 43*6777b538SAndroid Build Coastguard Worker test_filtering.add_cmdline_args(argparse_parser) 44*6777b538SAndroid Build Coastguard Worker test_results.add_cmdline_args(argparse_parser) 45*6777b538SAndroid Build Coastguard Worker 46*6777b538SAndroid Build Coastguard Worker argparse_parser.add_argument( 47*6777b538SAndroid Build Coastguard Worker '--isolated-script-test-launcher-retry-limit', 48*6777b538SAndroid Build Coastguard Worker dest='retry_limit', 49*6777b538SAndroid Build Coastguard Worker default=3, 50*6777b538SAndroid Build Coastguard Worker help='Sets the limit of test retries on failures to N.', 51*6777b538SAndroid Build Coastguard Worker metavar='N', 52*6777b538SAndroid Build Coastguard Worker type=int) 53*6777b538SAndroid Build Coastguard Worker argparse_parser.add_argument('--isolated-script-test-repeat', 54*6777b538SAndroid Build Coastguard Worker dest='repetitions', 55*6777b538SAndroid Build Coastguard Worker default=1, 56*6777b538SAndroid Build Coastguard Worker help='Repeats each test N times.', 57*6777b538SAndroid Build Coastguard Worker metavar='N', 58*6777b538SAndroid Build Coastguard Worker type=int) 59*6777b538SAndroid Build Coastguard Worker 60*6777b538SAndroid Build Coastguard Worker 61*6777b538SAndroid Build Coastguard Workerdef _calculate_tests_to_run(argparse_parsed_args, env, 62*6777b538SAndroid Build Coastguard Worker test_executable_wrappers): 63*6777b538SAndroid Build Coastguard Worker tests = [] 64*6777b538SAndroid Build Coastguard Worker for wrapper in test_executable_wrappers: 65*6777b538SAndroid Build Coastguard Worker extra_tests = wrapper.list_all_tests() 66*6777b538SAndroid Build Coastguard Worker for extra_test in extra_tests: 67*6777b538SAndroid Build Coastguard Worker assert extra_test not in tests 68*6777b538SAndroid Build Coastguard Worker tests.extend(extra_tests) 69*6777b538SAndroid Build Coastguard Worker return test_filtering.filter_tests(argparse_parsed_args, env, tests) 70*6777b538SAndroid Build Coastguard Worker 71*6777b538SAndroid Build Coastguard Worker 72*6777b538SAndroid Build Coastguard Workerdef _run_tests_and_save_results(argparse_parsed_args, list_of_tests_to_run, 73*6777b538SAndroid Build Coastguard Worker test_executable_wrapper): 74*6777b538SAndroid Build Coastguard Worker start_time = time.time() 75*6777b538SAndroid Build Coastguard Worker results = [] 76*6777b538SAndroid Build Coastguard Worker for wrapper in test_executable_wrapper: 77*6777b538SAndroid Build Coastguard Worker results.extend(wrapper.run_tests(list_of_tests_to_run)) 78*6777b538SAndroid Build Coastguard Worker test_results.print_test_results(argparse_parsed_args, results, start_time) 79*6777b538SAndroid Build Coastguard Worker 80*6777b538SAndroid Build Coastguard Worker 81*6777b538SAndroid Build Coastguard Workerdef main(test_executable_wrappers, argparse_parsed_args, env): 82*6777b538SAndroid Build Coastguard Worker """Runs tests within `test_executable_wrappers` using cmdline arguments and 83*6777b538SAndroid Build Coastguard Worker environment variables to figure out 1) which subset of tests to run, 2) 84*6777b538SAndroid Build Coastguard Worker where to save the JSON file with test results. 85*6777b538SAndroid Build Coastguard Worker 86*6777b538SAndroid Build Coastguard Worker Args: 87*6777b538SAndroid Build Coastguard Worker test_executable_wrappers: A list of objects providing 88*6777b538SAndroid Build Coastguard Worker list_all_tests(...) and run_tests(...) methods (see 89*6777b538SAndroid Build Coastguard Worker rust_main_program._TestExecutableWrapper). 90*6777b538SAndroid Build Coastguard Worker argparse_parsed_arg: A result of an earlier call to 91*6777b538SAndroid Build Coastguard Worker argparse_parser.parse_args() call (where `argparse_parser` has been 92*6777b538SAndroid Build Coastguard Worker populated via an even earlier call to add_cmdline_args). 93*6777b538SAndroid Build Coastguard Worker env: a dictionary-like object (typically from `os.environ`). 94*6777b538SAndroid Build Coastguard Worker """ 95*6777b538SAndroid Build Coastguard Worker list_of_test_names_to_run = _calculate_tests_to_run( 96*6777b538SAndroid Build Coastguard Worker argparse_parsed_args, env, test_executable_wrappers) 97*6777b538SAndroid Build Coastguard Worker _run_tests_and_save_results(argparse_parsed_args, 98*6777b538SAndroid Build Coastguard Worker list_of_test_names_to_run, 99*6777b538SAndroid Build Coastguard Worker test_executable_wrappers) 100*6777b538SAndroid Build Coastguard Worker # TODO(lukasza): Repeat tests `args.repetitions` times. 101*6777b538SAndroid Build Coastguard Worker # TODO(lukasza): Retry failing times up to `args.retry_limit` times. 102