1*dbb99499SAndroid Build Coastguard Worker#!/usr/bin/env python3 2*dbb99499SAndroid Build Coastguard Worker 3*dbb99499SAndroid Build Coastguard Worker# type: ignore 4*dbb99499SAndroid Build Coastguard Worker 5*dbb99499SAndroid Build Coastguard Worker""" 6*dbb99499SAndroid Build Coastguard Workercompare.py - versatile benchmark output compare tool 7*dbb99499SAndroid Build Coastguard Worker""" 8*dbb99499SAndroid Build Coastguard Worker 9*dbb99499SAndroid Build Coastguard Workerimport argparse 10*dbb99499SAndroid Build Coastguard Workerimport json 11*dbb99499SAndroid Build Coastguard Workerimport os 12*dbb99499SAndroid Build Coastguard Workerimport sys 13*dbb99499SAndroid Build Coastguard Workerimport unittest 14*dbb99499SAndroid Build Coastguard Workerfrom argparse import ArgumentParser 15*dbb99499SAndroid Build Coastguard Worker 16*dbb99499SAndroid Build Coastguard Workerimport gbench 17*dbb99499SAndroid Build Coastguard Workerfrom gbench import report, util 18*dbb99499SAndroid Build Coastguard Worker 19*dbb99499SAndroid Build Coastguard Worker 20*dbb99499SAndroid Build Coastguard Workerdef check_inputs(in1, in2, flags): 21*dbb99499SAndroid Build Coastguard Worker """ 22*dbb99499SAndroid Build Coastguard Worker Perform checking on the user provided inputs and diagnose any abnormalities 23*dbb99499SAndroid Build Coastguard Worker """ 24*dbb99499SAndroid Build Coastguard Worker in1_kind, in1_err = util.classify_input_file(in1) 25*dbb99499SAndroid Build Coastguard Worker in2_kind, in2_err = util.classify_input_file(in2) 26*dbb99499SAndroid Build Coastguard Worker output_file = util.find_benchmark_flag("--benchmark_out=", flags) 27*dbb99499SAndroid Build Coastguard Worker output_type = util.find_benchmark_flag("--benchmark_out_format=", flags) 28*dbb99499SAndroid Build Coastguard Worker if ( 29*dbb99499SAndroid Build Coastguard Worker in1_kind == util.IT_Executable 30*dbb99499SAndroid Build Coastguard Worker and in2_kind == util.IT_Executable 31*dbb99499SAndroid Build Coastguard Worker and output_file 32*dbb99499SAndroid Build Coastguard Worker ): 33*dbb99499SAndroid Build Coastguard Worker print( 34*dbb99499SAndroid Build Coastguard Worker ( 35*dbb99499SAndroid Build Coastguard Worker "WARNING: '--benchmark_out=%s' will be passed to both " 36*dbb99499SAndroid Build Coastguard Worker "benchmarks causing it to be overwritten" 37*dbb99499SAndroid Build Coastguard Worker ) 38*dbb99499SAndroid Build Coastguard Worker % output_file 39*dbb99499SAndroid Build Coastguard Worker ) 40*dbb99499SAndroid Build Coastguard Worker if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON: 41*dbb99499SAndroid Build Coastguard Worker # When both sides are JSON the only supported flag is 42*dbb99499SAndroid Build Coastguard Worker # --benchmark_filter= 43*dbb99499SAndroid Build Coastguard Worker for flag in util.remove_benchmark_flags("--benchmark_filter=", flags): 44*dbb99499SAndroid Build Coastguard Worker print( 45*dbb99499SAndroid Build Coastguard Worker "WARNING: passing %s has no effect since both " 46*dbb99499SAndroid Build Coastguard Worker "inputs are JSON" % flag 47*dbb99499SAndroid Build Coastguard Worker ) 48*dbb99499SAndroid Build Coastguard Worker if output_type is not None and output_type != "json": 49*dbb99499SAndroid Build Coastguard Worker print( 50*dbb99499SAndroid Build Coastguard Worker ( 51*dbb99499SAndroid Build Coastguard Worker "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`" 52*dbb99499SAndroid Build Coastguard Worker " is not supported." 53*dbb99499SAndroid Build Coastguard Worker ) 54*dbb99499SAndroid Build Coastguard Worker % output_type 55*dbb99499SAndroid Build Coastguard Worker ) 56*dbb99499SAndroid Build Coastguard Worker sys.exit(1) 57*dbb99499SAndroid Build Coastguard Worker 58*dbb99499SAndroid Build Coastguard Worker 59*dbb99499SAndroid Build Coastguard Workerdef create_parser(): 60*dbb99499SAndroid Build Coastguard Worker parser = ArgumentParser( 61*dbb99499SAndroid Build Coastguard Worker description="versatile benchmark output compare tool" 62*dbb99499SAndroid Build Coastguard Worker ) 63*dbb99499SAndroid Build Coastguard Worker 64*dbb99499SAndroid Build Coastguard Worker parser.add_argument( 65*dbb99499SAndroid Build Coastguard Worker "-a", 66*dbb99499SAndroid Build Coastguard Worker "--display_aggregates_only", 67*dbb99499SAndroid Build Coastguard Worker dest="display_aggregates_only", 68*dbb99499SAndroid Build Coastguard Worker action="store_true", 69*dbb99499SAndroid Build Coastguard Worker help="If there are repetitions, by default, we display everything - the" 70*dbb99499SAndroid Build Coastguard Worker " actual runs, and the aggregates computed. Sometimes, it is " 71*dbb99499SAndroid Build Coastguard Worker "desirable to only view the aggregates. E.g. when there are a lot " 72*dbb99499SAndroid Build Coastguard Worker "of repetitions. Do note that only the display is affected. " 73*dbb99499SAndroid Build Coastguard Worker "Internally, all the actual runs are still used, e.g. for U test.", 74*dbb99499SAndroid Build Coastguard Worker ) 75*dbb99499SAndroid Build Coastguard Worker 76*dbb99499SAndroid Build Coastguard Worker parser.add_argument( 77*dbb99499SAndroid Build Coastguard Worker "--no-color", 78*dbb99499SAndroid Build Coastguard Worker dest="color", 79*dbb99499SAndroid Build Coastguard Worker default=True, 80*dbb99499SAndroid Build Coastguard Worker action="store_false", 81*dbb99499SAndroid Build Coastguard Worker help="Do not use colors in the terminal output", 82*dbb99499SAndroid Build Coastguard Worker ) 83*dbb99499SAndroid Build Coastguard Worker 84*dbb99499SAndroid Build Coastguard Worker parser.add_argument( 85*dbb99499SAndroid Build Coastguard Worker "-d", 86*dbb99499SAndroid Build Coastguard Worker "--dump_to_json", 87*dbb99499SAndroid Build Coastguard Worker dest="dump_to_json", 88*dbb99499SAndroid Build Coastguard Worker help="Additionally, dump benchmark comparison output to this file in JSON format.", 89*dbb99499SAndroid Build Coastguard Worker ) 90*dbb99499SAndroid Build Coastguard Worker 91*dbb99499SAndroid Build Coastguard Worker utest = parser.add_argument_group() 92*dbb99499SAndroid Build Coastguard Worker utest.add_argument( 93*dbb99499SAndroid Build Coastguard Worker "--no-utest", 94*dbb99499SAndroid Build Coastguard Worker dest="utest", 95*dbb99499SAndroid Build Coastguard Worker default=True, 96*dbb99499SAndroid Build Coastguard Worker action="store_false", 97*dbb99499SAndroid Build Coastguard Worker help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format( 98*dbb99499SAndroid Build Coastguard Worker report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS 99*dbb99499SAndroid Build Coastguard Worker ), 100*dbb99499SAndroid Build Coastguard Worker ) 101*dbb99499SAndroid Build Coastguard Worker alpha_default = 0.05 102*dbb99499SAndroid Build Coastguard Worker utest.add_argument( 103*dbb99499SAndroid Build Coastguard Worker "--alpha", 104*dbb99499SAndroid Build Coastguard Worker dest="utest_alpha", 105*dbb99499SAndroid Build Coastguard Worker default=alpha_default, 106*dbb99499SAndroid Build Coastguard Worker type=float, 107*dbb99499SAndroid Build Coastguard Worker help=( 108*dbb99499SAndroid Build Coastguard Worker "significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)" 109*dbb99499SAndroid Build Coastguard Worker ) 110*dbb99499SAndroid Build Coastguard Worker % alpha_default, 111*dbb99499SAndroid Build Coastguard Worker ) 112*dbb99499SAndroid Build Coastguard Worker 113*dbb99499SAndroid Build Coastguard Worker subparsers = parser.add_subparsers( 114*dbb99499SAndroid Build Coastguard Worker help="This tool has multiple modes of operation:", dest="mode" 115*dbb99499SAndroid Build Coastguard Worker ) 116*dbb99499SAndroid Build Coastguard Worker 117*dbb99499SAndroid Build Coastguard Worker parser_a = subparsers.add_parser( 118*dbb99499SAndroid Build Coastguard Worker "benchmarks", 119*dbb99499SAndroid Build Coastguard Worker help="The most simple use-case, compare all the output of these two benchmarks", 120*dbb99499SAndroid Build Coastguard Worker ) 121*dbb99499SAndroid Build Coastguard Worker baseline = parser_a.add_argument_group("baseline", "The benchmark baseline") 122*dbb99499SAndroid Build Coastguard Worker baseline.add_argument( 123*dbb99499SAndroid Build Coastguard Worker "test_baseline", 124*dbb99499SAndroid Build Coastguard Worker metavar="test_baseline", 125*dbb99499SAndroid Build Coastguard Worker type=argparse.FileType("r"), 126*dbb99499SAndroid Build Coastguard Worker nargs=1, 127*dbb99499SAndroid Build Coastguard Worker help="A benchmark executable or JSON output file", 128*dbb99499SAndroid Build Coastguard Worker ) 129*dbb99499SAndroid Build Coastguard Worker contender = parser_a.add_argument_group( 130*dbb99499SAndroid Build Coastguard Worker "contender", "The benchmark that will be compared against the baseline" 131*dbb99499SAndroid Build Coastguard Worker ) 132*dbb99499SAndroid Build Coastguard Worker contender.add_argument( 133*dbb99499SAndroid Build Coastguard Worker "test_contender", 134*dbb99499SAndroid Build Coastguard Worker metavar="test_contender", 135*dbb99499SAndroid Build Coastguard Worker type=argparse.FileType("r"), 136*dbb99499SAndroid Build Coastguard Worker nargs=1, 137*dbb99499SAndroid Build Coastguard Worker help="A benchmark executable or JSON output file", 138*dbb99499SAndroid Build Coastguard Worker ) 139*dbb99499SAndroid Build Coastguard Worker parser_a.add_argument( 140*dbb99499SAndroid Build Coastguard Worker "benchmark_options", 141*dbb99499SAndroid Build Coastguard Worker metavar="benchmark_options", 142*dbb99499SAndroid Build Coastguard Worker nargs=argparse.REMAINDER, 143*dbb99499SAndroid Build Coastguard Worker help="Arguments to pass when running benchmark executables", 144*dbb99499SAndroid Build Coastguard Worker ) 145*dbb99499SAndroid Build Coastguard Worker 146*dbb99499SAndroid Build Coastguard Worker parser_b = subparsers.add_parser( 147*dbb99499SAndroid Build Coastguard Worker "filters", help="Compare filter one with the filter two of benchmark" 148*dbb99499SAndroid Build Coastguard Worker ) 149*dbb99499SAndroid Build Coastguard Worker baseline = parser_b.add_argument_group("baseline", "The benchmark baseline") 150*dbb99499SAndroid Build Coastguard Worker baseline.add_argument( 151*dbb99499SAndroid Build Coastguard Worker "test", 152*dbb99499SAndroid Build Coastguard Worker metavar="test", 153*dbb99499SAndroid Build Coastguard Worker type=argparse.FileType("r"), 154*dbb99499SAndroid Build Coastguard Worker nargs=1, 155*dbb99499SAndroid Build Coastguard Worker help="A benchmark executable or JSON output file", 156*dbb99499SAndroid Build Coastguard Worker ) 157*dbb99499SAndroid Build Coastguard Worker baseline.add_argument( 158*dbb99499SAndroid Build Coastguard Worker "filter_baseline", 159*dbb99499SAndroid Build Coastguard Worker metavar="filter_baseline", 160*dbb99499SAndroid Build Coastguard Worker type=str, 161*dbb99499SAndroid Build Coastguard Worker nargs=1, 162*dbb99499SAndroid Build Coastguard Worker help="The first filter, that will be used as baseline", 163*dbb99499SAndroid Build Coastguard Worker ) 164*dbb99499SAndroid Build Coastguard Worker contender = parser_b.add_argument_group( 165*dbb99499SAndroid Build Coastguard Worker "contender", "The benchmark that will be compared against the baseline" 166*dbb99499SAndroid Build Coastguard Worker ) 167*dbb99499SAndroid Build Coastguard Worker contender.add_argument( 168*dbb99499SAndroid Build Coastguard Worker "filter_contender", 169*dbb99499SAndroid Build Coastguard Worker metavar="filter_contender", 170*dbb99499SAndroid Build Coastguard Worker type=str, 171*dbb99499SAndroid Build Coastguard Worker nargs=1, 172*dbb99499SAndroid Build Coastguard Worker help="The second filter, that will be compared against the baseline", 173*dbb99499SAndroid Build Coastguard Worker ) 174*dbb99499SAndroid Build Coastguard Worker parser_b.add_argument( 175*dbb99499SAndroid Build Coastguard Worker "benchmark_options", 176*dbb99499SAndroid Build Coastguard Worker metavar="benchmark_options", 177*dbb99499SAndroid Build Coastguard Worker nargs=argparse.REMAINDER, 178*dbb99499SAndroid Build Coastguard Worker help="Arguments to pass when running benchmark executables", 179*dbb99499SAndroid Build Coastguard Worker ) 180*dbb99499SAndroid Build Coastguard Worker 181*dbb99499SAndroid Build Coastguard Worker parser_c = subparsers.add_parser( 182*dbb99499SAndroid Build Coastguard Worker "benchmarksfiltered", 183*dbb99499SAndroid Build Coastguard Worker help="Compare filter one of first benchmark with filter two of the second benchmark", 184*dbb99499SAndroid Build Coastguard Worker ) 185*dbb99499SAndroid Build Coastguard Worker baseline = parser_c.add_argument_group("baseline", "The benchmark baseline") 186*dbb99499SAndroid Build Coastguard Worker baseline.add_argument( 187*dbb99499SAndroid Build Coastguard Worker "test_baseline", 188*dbb99499SAndroid Build Coastguard Worker metavar="test_baseline", 189*dbb99499SAndroid Build Coastguard Worker type=argparse.FileType("r"), 190*dbb99499SAndroid Build Coastguard Worker nargs=1, 191*dbb99499SAndroid Build Coastguard Worker help="A benchmark executable or JSON output file", 192*dbb99499SAndroid Build Coastguard Worker ) 193*dbb99499SAndroid Build Coastguard Worker baseline.add_argument( 194*dbb99499SAndroid Build Coastguard Worker "filter_baseline", 195*dbb99499SAndroid Build Coastguard Worker metavar="filter_baseline", 196*dbb99499SAndroid Build Coastguard Worker type=str, 197*dbb99499SAndroid Build Coastguard Worker nargs=1, 198*dbb99499SAndroid Build Coastguard Worker help="The first filter, that will be used as baseline", 199*dbb99499SAndroid Build Coastguard Worker ) 200*dbb99499SAndroid Build Coastguard Worker contender = parser_c.add_argument_group( 201*dbb99499SAndroid Build Coastguard Worker "contender", "The benchmark that will be compared against the baseline" 202*dbb99499SAndroid Build Coastguard Worker ) 203*dbb99499SAndroid Build Coastguard Worker contender.add_argument( 204*dbb99499SAndroid Build Coastguard Worker "test_contender", 205*dbb99499SAndroid Build Coastguard Worker metavar="test_contender", 206*dbb99499SAndroid Build Coastguard Worker type=argparse.FileType("r"), 207*dbb99499SAndroid Build Coastguard Worker nargs=1, 208*dbb99499SAndroid Build Coastguard Worker help="The second benchmark executable or JSON output file, that will be compared against the baseline", 209*dbb99499SAndroid Build Coastguard Worker ) 210*dbb99499SAndroid Build Coastguard Worker contender.add_argument( 211*dbb99499SAndroid Build Coastguard Worker "filter_contender", 212*dbb99499SAndroid Build Coastguard Worker metavar="filter_contender", 213*dbb99499SAndroid Build Coastguard Worker type=str, 214*dbb99499SAndroid Build Coastguard Worker nargs=1, 215*dbb99499SAndroid Build Coastguard Worker help="The second filter, that will be compared against the baseline", 216*dbb99499SAndroid Build Coastguard Worker ) 217*dbb99499SAndroid Build Coastguard Worker parser_c.add_argument( 218*dbb99499SAndroid Build Coastguard Worker "benchmark_options", 219*dbb99499SAndroid Build Coastguard Worker metavar="benchmark_options", 220*dbb99499SAndroid Build Coastguard Worker nargs=argparse.REMAINDER, 221*dbb99499SAndroid Build Coastguard Worker help="Arguments to pass when running benchmark executables", 222*dbb99499SAndroid Build Coastguard Worker ) 223*dbb99499SAndroid Build Coastguard Worker 224*dbb99499SAndroid Build Coastguard Worker return parser 225*dbb99499SAndroid Build Coastguard Worker 226*dbb99499SAndroid Build Coastguard Worker 227*dbb99499SAndroid Build Coastguard Workerdef main(): 228*dbb99499SAndroid Build Coastguard Worker # Parse the command line flags 229*dbb99499SAndroid Build Coastguard Worker parser = create_parser() 230*dbb99499SAndroid Build Coastguard Worker args, unknown_args = parser.parse_known_args() 231*dbb99499SAndroid Build Coastguard Worker if args.mode is None: 232*dbb99499SAndroid Build Coastguard Worker parser.print_help() 233*dbb99499SAndroid Build Coastguard Worker exit(1) 234*dbb99499SAndroid Build Coastguard Worker assert not unknown_args 235*dbb99499SAndroid Build Coastguard Worker benchmark_options = args.benchmark_options 236*dbb99499SAndroid Build Coastguard Worker 237*dbb99499SAndroid Build Coastguard Worker if args.mode == "benchmarks": 238*dbb99499SAndroid Build Coastguard Worker test_baseline = args.test_baseline[0].name 239*dbb99499SAndroid Build Coastguard Worker test_contender = args.test_contender[0].name 240*dbb99499SAndroid Build Coastguard Worker filter_baseline = "" 241*dbb99499SAndroid Build Coastguard Worker filter_contender = "" 242*dbb99499SAndroid Build Coastguard Worker 243*dbb99499SAndroid Build Coastguard Worker # NOTE: if test_baseline == test_contender, you are analyzing the stdev 244*dbb99499SAndroid Build Coastguard Worker 245*dbb99499SAndroid Build Coastguard Worker description = "Comparing %s to %s" % (test_baseline, test_contender) 246*dbb99499SAndroid Build Coastguard Worker elif args.mode == "filters": 247*dbb99499SAndroid Build Coastguard Worker test_baseline = args.test[0].name 248*dbb99499SAndroid Build Coastguard Worker test_contender = args.test[0].name 249*dbb99499SAndroid Build Coastguard Worker filter_baseline = args.filter_baseline[0] 250*dbb99499SAndroid Build Coastguard Worker filter_contender = args.filter_contender[0] 251*dbb99499SAndroid Build Coastguard Worker 252*dbb99499SAndroid Build Coastguard Worker # NOTE: if filter_baseline == filter_contender, you are analyzing the 253*dbb99499SAndroid Build Coastguard Worker # stdev 254*dbb99499SAndroid Build Coastguard Worker 255*dbb99499SAndroid Build Coastguard Worker description = "Comparing %s to %s (from %s)" % ( 256*dbb99499SAndroid Build Coastguard Worker filter_baseline, 257*dbb99499SAndroid Build Coastguard Worker filter_contender, 258*dbb99499SAndroid Build Coastguard Worker args.test[0].name, 259*dbb99499SAndroid Build Coastguard Worker ) 260*dbb99499SAndroid Build Coastguard Worker elif args.mode == "benchmarksfiltered": 261*dbb99499SAndroid Build Coastguard Worker test_baseline = args.test_baseline[0].name 262*dbb99499SAndroid Build Coastguard Worker test_contender = args.test_contender[0].name 263*dbb99499SAndroid Build Coastguard Worker filter_baseline = args.filter_baseline[0] 264*dbb99499SAndroid Build Coastguard Worker filter_contender = args.filter_contender[0] 265*dbb99499SAndroid Build Coastguard Worker 266*dbb99499SAndroid Build Coastguard Worker # NOTE: if test_baseline == test_contender and 267*dbb99499SAndroid Build Coastguard Worker # filter_baseline == filter_contender, you are analyzing the stdev 268*dbb99499SAndroid Build Coastguard Worker 269*dbb99499SAndroid Build Coastguard Worker description = "Comparing %s (from %s) to %s (from %s)" % ( 270*dbb99499SAndroid Build Coastguard Worker filter_baseline, 271*dbb99499SAndroid Build Coastguard Worker test_baseline, 272*dbb99499SAndroid Build Coastguard Worker filter_contender, 273*dbb99499SAndroid Build Coastguard Worker test_contender, 274*dbb99499SAndroid Build Coastguard Worker ) 275*dbb99499SAndroid Build Coastguard Worker else: 276*dbb99499SAndroid Build Coastguard Worker # should never happen 277*dbb99499SAndroid Build Coastguard Worker print("Unrecognized mode of operation: '%s'" % args.mode) 278*dbb99499SAndroid Build Coastguard Worker parser.print_help() 279*dbb99499SAndroid Build Coastguard Worker exit(1) 280*dbb99499SAndroid Build Coastguard Worker 281*dbb99499SAndroid Build Coastguard Worker check_inputs(test_baseline, test_contender, benchmark_options) 282*dbb99499SAndroid Build Coastguard Worker 283*dbb99499SAndroid Build Coastguard Worker if args.display_aggregates_only: 284*dbb99499SAndroid Build Coastguard Worker benchmark_options += ["--benchmark_display_aggregates_only=true"] 285*dbb99499SAndroid Build Coastguard Worker 286*dbb99499SAndroid Build Coastguard Worker options_baseline = [] 287*dbb99499SAndroid Build Coastguard Worker options_contender = [] 288*dbb99499SAndroid Build Coastguard Worker 289*dbb99499SAndroid Build Coastguard Worker if filter_baseline and filter_contender: 290*dbb99499SAndroid Build Coastguard Worker options_baseline = ["--benchmark_filter=%s" % filter_baseline] 291*dbb99499SAndroid Build Coastguard Worker options_contender = ["--benchmark_filter=%s" % filter_contender] 292*dbb99499SAndroid Build Coastguard Worker 293*dbb99499SAndroid Build Coastguard Worker # Run the benchmarks and report the results 294*dbb99499SAndroid Build Coastguard Worker json1 = json1_orig = gbench.util.sort_benchmark_results( 295*dbb99499SAndroid Build Coastguard Worker gbench.util.run_or_load_benchmark( 296*dbb99499SAndroid Build Coastguard Worker test_baseline, benchmark_options + options_baseline 297*dbb99499SAndroid Build Coastguard Worker ) 298*dbb99499SAndroid Build Coastguard Worker ) 299*dbb99499SAndroid Build Coastguard Worker json2 = json2_orig = gbench.util.sort_benchmark_results( 300*dbb99499SAndroid Build Coastguard Worker gbench.util.run_or_load_benchmark( 301*dbb99499SAndroid Build Coastguard Worker test_contender, benchmark_options + options_contender 302*dbb99499SAndroid Build Coastguard Worker ) 303*dbb99499SAndroid Build Coastguard Worker ) 304*dbb99499SAndroid Build Coastguard Worker 305*dbb99499SAndroid Build Coastguard Worker # Now, filter the benchmarks so that the difference report can work 306*dbb99499SAndroid Build Coastguard Worker if filter_baseline and filter_contender: 307*dbb99499SAndroid Build Coastguard Worker replacement = "[%s vs. %s]" % (filter_baseline, filter_contender) 308*dbb99499SAndroid Build Coastguard Worker json1 = gbench.report.filter_benchmark( 309*dbb99499SAndroid Build Coastguard Worker json1_orig, filter_baseline, replacement 310*dbb99499SAndroid Build Coastguard Worker ) 311*dbb99499SAndroid Build Coastguard Worker json2 = gbench.report.filter_benchmark( 312*dbb99499SAndroid Build Coastguard Worker json2_orig, filter_contender, replacement 313*dbb99499SAndroid Build Coastguard Worker ) 314*dbb99499SAndroid Build Coastguard Worker 315*dbb99499SAndroid Build Coastguard Worker diff_report = gbench.report.get_difference_report(json1, json2, args.utest) 316*dbb99499SAndroid Build Coastguard Worker output_lines = gbench.report.print_difference_report( 317*dbb99499SAndroid Build Coastguard Worker diff_report, 318*dbb99499SAndroid Build Coastguard Worker args.display_aggregates_only, 319*dbb99499SAndroid Build Coastguard Worker args.utest, 320*dbb99499SAndroid Build Coastguard Worker args.utest_alpha, 321*dbb99499SAndroid Build Coastguard Worker args.color, 322*dbb99499SAndroid Build Coastguard Worker ) 323*dbb99499SAndroid Build Coastguard Worker print(description) 324*dbb99499SAndroid Build Coastguard Worker for ln in output_lines: 325*dbb99499SAndroid Build Coastguard Worker print(ln) 326*dbb99499SAndroid Build Coastguard Worker 327*dbb99499SAndroid Build Coastguard Worker # Optionally, diff and output to JSON 328*dbb99499SAndroid Build Coastguard Worker if args.dump_to_json is not None: 329*dbb99499SAndroid Build Coastguard Worker with open(args.dump_to_json, "w") as f_json: 330*dbb99499SAndroid Build Coastguard Worker json.dump(diff_report, f_json, indent=1) 331*dbb99499SAndroid Build Coastguard Worker 332*dbb99499SAndroid Build Coastguard Worker 333*dbb99499SAndroid Build Coastguard Workerclass TestParser(unittest.TestCase): 334*dbb99499SAndroid Build Coastguard Worker def setUp(self): 335*dbb99499SAndroid Build Coastguard Worker self.parser = create_parser() 336*dbb99499SAndroid Build Coastguard Worker testInputs = os.path.join( 337*dbb99499SAndroid Build Coastguard Worker os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs" 338*dbb99499SAndroid Build Coastguard Worker ) 339*dbb99499SAndroid Build Coastguard Worker self.testInput0 = os.path.join(testInputs, "test1_run1.json") 340*dbb99499SAndroid Build Coastguard Worker self.testInput1 = os.path.join(testInputs, "test1_run2.json") 341*dbb99499SAndroid Build Coastguard Worker 342*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_basic(self): 343*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 344*dbb99499SAndroid Build Coastguard Worker ["benchmarks", self.testInput0, self.testInput1] 345*dbb99499SAndroid Build Coastguard Worker ) 346*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 347*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 348*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 349*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 350*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 351*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 352*dbb99499SAndroid Build Coastguard Worker 353*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_basic_without_utest(self): 354*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 355*dbb99499SAndroid Build Coastguard Worker ["--no-utest", "benchmarks", self.testInput0, self.testInput1] 356*dbb99499SAndroid Build Coastguard Worker ) 357*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 358*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.utest) 359*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.utest_alpha, 0.05) 360*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 361*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 362*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 363*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 364*dbb99499SAndroid Build Coastguard Worker 365*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_basic_display_aggregates_only(self): 366*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 367*dbb99499SAndroid Build Coastguard Worker ["-a", "benchmarks", self.testInput0, self.testInput1] 368*dbb99499SAndroid Build Coastguard Worker ) 369*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.display_aggregates_only) 370*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 371*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 372*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 373*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 374*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 375*dbb99499SAndroid Build Coastguard Worker 376*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_basic_with_utest_alpha(self): 377*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 378*dbb99499SAndroid Build Coastguard Worker ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1] 379*dbb99499SAndroid Build Coastguard Worker ) 380*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 381*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 382*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.utest_alpha, 0.314) 383*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 384*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 385*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 386*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 387*dbb99499SAndroid Build Coastguard Worker 388*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_basic_without_utest_with_utest_alpha(self): 389*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 390*dbb99499SAndroid Build Coastguard Worker [ 391*dbb99499SAndroid Build Coastguard Worker "--no-utest", 392*dbb99499SAndroid Build Coastguard Worker "--alpha=0.314", 393*dbb99499SAndroid Build Coastguard Worker "benchmarks", 394*dbb99499SAndroid Build Coastguard Worker self.testInput0, 395*dbb99499SAndroid Build Coastguard Worker self.testInput1, 396*dbb99499SAndroid Build Coastguard Worker ] 397*dbb99499SAndroid Build Coastguard Worker ) 398*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 399*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.utest) 400*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.utest_alpha, 0.314) 401*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 402*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 403*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 404*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 405*dbb99499SAndroid Build Coastguard Worker 406*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_with_remainder(self): 407*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 408*dbb99499SAndroid Build Coastguard Worker ["benchmarks", self.testInput0, self.testInput1, "d"] 409*dbb99499SAndroid Build Coastguard Worker ) 410*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 411*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 412*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 413*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 414*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 415*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options, ["d"]) 416*dbb99499SAndroid Build Coastguard Worker 417*dbb99499SAndroid Build Coastguard Worker def test_benchmarks_with_remainder_after_doubleminus(self): 418*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 419*dbb99499SAndroid Build Coastguard Worker ["benchmarks", self.testInput0, self.testInput1, "--", "e"] 420*dbb99499SAndroid Build Coastguard Worker ) 421*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 422*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 423*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarks") 424*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 425*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 426*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options, ["e"]) 427*dbb99499SAndroid Build Coastguard Worker 428*dbb99499SAndroid Build Coastguard Worker def test_filters_basic(self): 429*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"]) 430*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 431*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 432*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "filters") 433*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test[0].name, self.testInput0) 434*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 435*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "d") 436*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 437*dbb99499SAndroid Build Coastguard Worker 438*dbb99499SAndroid Build Coastguard Worker def test_filters_with_remainder(self): 439*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 440*dbb99499SAndroid Build Coastguard Worker ["filters", self.testInput0, "c", "d", "e"] 441*dbb99499SAndroid Build Coastguard Worker ) 442*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 443*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 444*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "filters") 445*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test[0].name, self.testInput0) 446*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 447*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "d") 448*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options, ["e"]) 449*dbb99499SAndroid Build Coastguard Worker 450*dbb99499SAndroid Build Coastguard Worker def test_filters_with_remainder_after_doubleminus(self): 451*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 452*dbb99499SAndroid Build Coastguard Worker ["filters", self.testInput0, "c", "d", "--", "f"] 453*dbb99499SAndroid Build Coastguard Worker ) 454*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 455*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 456*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "filters") 457*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test[0].name, self.testInput0) 458*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 459*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "d") 460*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options, ["f"]) 461*dbb99499SAndroid Build Coastguard Worker 462*dbb99499SAndroid Build Coastguard Worker def test_benchmarksfiltered_basic(self): 463*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 464*dbb99499SAndroid Build Coastguard Worker ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"] 465*dbb99499SAndroid Build Coastguard Worker ) 466*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 467*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 468*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarksfiltered") 469*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 470*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 471*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 472*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "e") 473*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.benchmark_options) 474*dbb99499SAndroid Build Coastguard Worker 475*dbb99499SAndroid Build Coastguard Worker def test_benchmarksfiltered_with_remainder(self): 476*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 477*dbb99499SAndroid Build Coastguard Worker [ 478*dbb99499SAndroid Build Coastguard Worker "benchmarksfiltered", 479*dbb99499SAndroid Build Coastguard Worker self.testInput0, 480*dbb99499SAndroid Build Coastguard Worker "c", 481*dbb99499SAndroid Build Coastguard Worker self.testInput1, 482*dbb99499SAndroid Build Coastguard Worker "e", 483*dbb99499SAndroid Build Coastguard Worker "f", 484*dbb99499SAndroid Build Coastguard Worker ] 485*dbb99499SAndroid Build Coastguard Worker ) 486*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 487*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 488*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarksfiltered") 489*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 490*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 491*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 492*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "e") 493*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options[0], "f") 494*dbb99499SAndroid Build Coastguard Worker 495*dbb99499SAndroid Build Coastguard Worker def test_benchmarksfiltered_with_remainder_after_doubleminus(self): 496*dbb99499SAndroid Build Coastguard Worker parsed = self.parser.parse_args( 497*dbb99499SAndroid Build Coastguard Worker [ 498*dbb99499SAndroid Build Coastguard Worker "benchmarksfiltered", 499*dbb99499SAndroid Build Coastguard Worker self.testInput0, 500*dbb99499SAndroid Build Coastguard Worker "c", 501*dbb99499SAndroid Build Coastguard Worker self.testInput1, 502*dbb99499SAndroid Build Coastguard Worker "e", 503*dbb99499SAndroid Build Coastguard Worker "--", 504*dbb99499SAndroid Build Coastguard Worker "g", 505*dbb99499SAndroid Build Coastguard Worker ] 506*dbb99499SAndroid Build Coastguard Worker ) 507*dbb99499SAndroid Build Coastguard Worker self.assertFalse(parsed.display_aggregates_only) 508*dbb99499SAndroid Build Coastguard Worker self.assertTrue(parsed.utest) 509*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.mode, "benchmarksfiltered") 510*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_baseline[0].name, self.testInput0) 511*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_baseline[0], "c") 512*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.test_contender[0].name, self.testInput1) 513*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.filter_contender[0], "e") 514*dbb99499SAndroid Build Coastguard Worker self.assertEqual(parsed.benchmark_options[0], "g") 515*dbb99499SAndroid Build Coastguard Worker 516*dbb99499SAndroid Build Coastguard Worker 517*dbb99499SAndroid Build Coastguard Workerif __name__ == "__main__": 518*dbb99499SAndroid Build Coastguard Worker # unittest.main() 519*dbb99499SAndroid Build Coastguard Worker main() 520*dbb99499SAndroid Build Coastguard Worker 521*dbb99499SAndroid Build Coastguard Worker# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4 522*dbb99499SAndroid Build Coastguard Worker# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off; 523*dbb99499SAndroid Build Coastguard Worker# kate: indent-mode python; remove-trailing-spaces modified; 524