xref: /aosp_15_r20/external/google-benchmark/tools/compare.py (revision dbb99499c3810fa1611fa2242a2fc446be01a57c)
1*dbb99499SAndroid Build Coastguard Worker#!/usr/bin/env python3
2*dbb99499SAndroid Build Coastguard Worker
3*dbb99499SAndroid Build Coastguard Worker# type: ignore
4*dbb99499SAndroid Build Coastguard Worker
5*dbb99499SAndroid Build Coastguard Worker"""
6*dbb99499SAndroid Build Coastguard Workercompare.py - versatile benchmark output compare tool
7*dbb99499SAndroid Build Coastguard Worker"""
8*dbb99499SAndroid Build Coastguard Worker
9*dbb99499SAndroid Build Coastguard Workerimport argparse
10*dbb99499SAndroid Build Coastguard Workerimport json
11*dbb99499SAndroid Build Coastguard Workerimport os
12*dbb99499SAndroid Build Coastguard Workerimport sys
13*dbb99499SAndroid Build Coastguard Workerimport unittest
14*dbb99499SAndroid Build Coastguard Workerfrom argparse import ArgumentParser
15*dbb99499SAndroid Build Coastguard Worker
16*dbb99499SAndroid Build Coastguard Workerimport gbench
17*dbb99499SAndroid Build Coastguard Workerfrom gbench import report, util
18*dbb99499SAndroid Build Coastguard Worker
19*dbb99499SAndroid Build Coastguard Worker
20*dbb99499SAndroid Build Coastguard Workerdef check_inputs(in1, in2, flags):
21*dbb99499SAndroid Build Coastguard Worker    """
22*dbb99499SAndroid Build Coastguard Worker    Perform checking on the user provided inputs and diagnose any abnormalities
23*dbb99499SAndroid Build Coastguard Worker    """
24*dbb99499SAndroid Build Coastguard Worker    in1_kind, in1_err = util.classify_input_file(in1)
25*dbb99499SAndroid Build Coastguard Worker    in2_kind, in2_err = util.classify_input_file(in2)
26*dbb99499SAndroid Build Coastguard Worker    output_file = util.find_benchmark_flag("--benchmark_out=", flags)
27*dbb99499SAndroid Build Coastguard Worker    output_type = util.find_benchmark_flag("--benchmark_out_format=", flags)
28*dbb99499SAndroid Build Coastguard Worker    if (
29*dbb99499SAndroid Build Coastguard Worker        in1_kind == util.IT_Executable
30*dbb99499SAndroid Build Coastguard Worker        and in2_kind == util.IT_Executable
31*dbb99499SAndroid Build Coastguard Worker        and output_file
32*dbb99499SAndroid Build Coastguard Worker    ):
33*dbb99499SAndroid Build Coastguard Worker        print(
34*dbb99499SAndroid Build Coastguard Worker            (
35*dbb99499SAndroid Build Coastguard Worker                "WARNING: '--benchmark_out=%s' will be passed to both "
36*dbb99499SAndroid Build Coastguard Worker                "benchmarks causing it to be overwritten"
37*dbb99499SAndroid Build Coastguard Worker            )
38*dbb99499SAndroid Build Coastguard Worker            % output_file
39*dbb99499SAndroid Build Coastguard Worker        )
40*dbb99499SAndroid Build Coastguard Worker    if in1_kind == util.IT_JSON and in2_kind == util.IT_JSON:
41*dbb99499SAndroid Build Coastguard Worker        # When both sides are JSON the only supported flag is
42*dbb99499SAndroid Build Coastguard Worker        # --benchmark_filter=
43*dbb99499SAndroid Build Coastguard Worker        for flag in util.remove_benchmark_flags("--benchmark_filter=", flags):
44*dbb99499SAndroid Build Coastguard Worker            print(
45*dbb99499SAndroid Build Coastguard Worker                "WARNING: passing %s has no effect since both "
46*dbb99499SAndroid Build Coastguard Worker                "inputs are JSON" % flag
47*dbb99499SAndroid Build Coastguard Worker            )
48*dbb99499SAndroid Build Coastguard Worker    if output_type is not None and output_type != "json":
49*dbb99499SAndroid Build Coastguard Worker        print(
50*dbb99499SAndroid Build Coastguard Worker            (
51*dbb99499SAndroid Build Coastguard Worker                "ERROR: passing '--benchmark_out_format=%s' to 'compare.py`"
52*dbb99499SAndroid Build Coastguard Worker                " is not supported."
53*dbb99499SAndroid Build Coastguard Worker            )
54*dbb99499SAndroid Build Coastguard Worker            % output_type
55*dbb99499SAndroid Build Coastguard Worker        )
56*dbb99499SAndroid Build Coastguard Worker        sys.exit(1)
57*dbb99499SAndroid Build Coastguard Worker
58*dbb99499SAndroid Build Coastguard Worker
59*dbb99499SAndroid Build Coastguard Workerdef create_parser():
60*dbb99499SAndroid Build Coastguard Worker    parser = ArgumentParser(
61*dbb99499SAndroid Build Coastguard Worker        description="versatile benchmark output compare tool"
62*dbb99499SAndroid Build Coastguard Worker    )
63*dbb99499SAndroid Build Coastguard Worker
64*dbb99499SAndroid Build Coastguard Worker    parser.add_argument(
65*dbb99499SAndroid Build Coastguard Worker        "-a",
66*dbb99499SAndroid Build Coastguard Worker        "--display_aggregates_only",
67*dbb99499SAndroid Build Coastguard Worker        dest="display_aggregates_only",
68*dbb99499SAndroid Build Coastguard Worker        action="store_true",
69*dbb99499SAndroid Build Coastguard Worker        help="If there are repetitions, by default, we display everything - the"
70*dbb99499SAndroid Build Coastguard Worker        " actual runs, and the aggregates computed. Sometimes, it is "
71*dbb99499SAndroid Build Coastguard Worker        "desirable to only view the aggregates. E.g. when there are a lot "
72*dbb99499SAndroid Build Coastguard Worker        "of repetitions. Do note that only the display is affected. "
73*dbb99499SAndroid Build Coastguard Worker        "Internally, all the actual runs are still used, e.g. for U test.",
74*dbb99499SAndroid Build Coastguard Worker    )
75*dbb99499SAndroid Build Coastguard Worker
76*dbb99499SAndroid Build Coastguard Worker    parser.add_argument(
77*dbb99499SAndroid Build Coastguard Worker        "--no-color",
78*dbb99499SAndroid Build Coastguard Worker        dest="color",
79*dbb99499SAndroid Build Coastguard Worker        default=True,
80*dbb99499SAndroid Build Coastguard Worker        action="store_false",
81*dbb99499SAndroid Build Coastguard Worker        help="Do not use colors in the terminal output",
82*dbb99499SAndroid Build Coastguard Worker    )
83*dbb99499SAndroid Build Coastguard Worker
84*dbb99499SAndroid Build Coastguard Worker    parser.add_argument(
85*dbb99499SAndroid Build Coastguard Worker        "-d",
86*dbb99499SAndroid Build Coastguard Worker        "--dump_to_json",
87*dbb99499SAndroid Build Coastguard Worker        dest="dump_to_json",
88*dbb99499SAndroid Build Coastguard Worker        help="Additionally, dump benchmark comparison output to this file in JSON format.",
89*dbb99499SAndroid Build Coastguard Worker    )
90*dbb99499SAndroid Build Coastguard Worker
91*dbb99499SAndroid Build Coastguard Worker    utest = parser.add_argument_group()
92*dbb99499SAndroid Build Coastguard Worker    utest.add_argument(
93*dbb99499SAndroid Build Coastguard Worker        "--no-utest",
94*dbb99499SAndroid Build Coastguard Worker        dest="utest",
95*dbb99499SAndroid Build Coastguard Worker        default=True,
96*dbb99499SAndroid Build Coastguard Worker        action="store_false",
97*dbb99499SAndroid Build Coastguard Worker        help="The tool can do a two-tailed Mann-Whitney U test with the null hypothesis that it is equally likely that a randomly selected value from one sample will be less than or greater than a randomly selected value from a second sample.\nWARNING: requires **LARGE** (no less than {}) number of repetitions to be meaningful!\nThe test is being done by default, if at least {} repetitions were done.\nThis option can disable the U Test.".format(
98*dbb99499SAndroid Build Coastguard Worker            report.UTEST_OPTIMAL_REPETITIONS, report.UTEST_MIN_REPETITIONS
99*dbb99499SAndroid Build Coastguard Worker        ),
100*dbb99499SAndroid Build Coastguard Worker    )
101*dbb99499SAndroid Build Coastguard Worker    alpha_default = 0.05
102*dbb99499SAndroid Build Coastguard Worker    utest.add_argument(
103*dbb99499SAndroid Build Coastguard Worker        "--alpha",
104*dbb99499SAndroid Build Coastguard Worker        dest="utest_alpha",
105*dbb99499SAndroid Build Coastguard Worker        default=alpha_default,
106*dbb99499SAndroid Build Coastguard Worker        type=float,
107*dbb99499SAndroid Build Coastguard Worker        help=(
108*dbb99499SAndroid Build Coastguard Worker            "significance level alpha. if the calculated p-value is below this value, then the result is said to be statistically significant and the null hypothesis is rejected.\n(default: %0.4f)"
109*dbb99499SAndroid Build Coastguard Worker        )
110*dbb99499SAndroid Build Coastguard Worker        % alpha_default,
111*dbb99499SAndroid Build Coastguard Worker    )
112*dbb99499SAndroid Build Coastguard Worker
113*dbb99499SAndroid Build Coastguard Worker    subparsers = parser.add_subparsers(
114*dbb99499SAndroid Build Coastguard Worker        help="This tool has multiple modes of operation:", dest="mode"
115*dbb99499SAndroid Build Coastguard Worker    )
116*dbb99499SAndroid Build Coastguard Worker
117*dbb99499SAndroid Build Coastguard Worker    parser_a = subparsers.add_parser(
118*dbb99499SAndroid Build Coastguard Worker        "benchmarks",
119*dbb99499SAndroid Build Coastguard Worker        help="The most simple use-case, compare all the output of these two benchmarks",
120*dbb99499SAndroid Build Coastguard Worker    )
121*dbb99499SAndroid Build Coastguard Worker    baseline = parser_a.add_argument_group("baseline", "The benchmark baseline")
122*dbb99499SAndroid Build Coastguard Worker    baseline.add_argument(
123*dbb99499SAndroid Build Coastguard Worker        "test_baseline",
124*dbb99499SAndroid Build Coastguard Worker        metavar="test_baseline",
125*dbb99499SAndroid Build Coastguard Worker        type=argparse.FileType("r"),
126*dbb99499SAndroid Build Coastguard Worker        nargs=1,
127*dbb99499SAndroid Build Coastguard Worker        help="A benchmark executable or JSON output file",
128*dbb99499SAndroid Build Coastguard Worker    )
129*dbb99499SAndroid Build Coastguard Worker    contender = parser_a.add_argument_group(
130*dbb99499SAndroid Build Coastguard Worker        "contender", "The benchmark that will be compared against the baseline"
131*dbb99499SAndroid Build Coastguard Worker    )
132*dbb99499SAndroid Build Coastguard Worker    contender.add_argument(
133*dbb99499SAndroid Build Coastguard Worker        "test_contender",
134*dbb99499SAndroid Build Coastguard Worker        metavar="test_contender",
135*dbb99499SAndroid Build Coastguard Worker        type=argparse.FileType("r"),
136*dbb99499SAndroid Build Coastguard Worker        nargs=1,
137*dbb99499SAndroid Build Coastguard Worker        help="A benchmark executable or JSON output file",
138*dbb99499SAndroid Build Coastguard Worker    )
139*dbb99499SAndroid Build Coastguard Worker    parser_a.add_argument(
140*dbb99499SAndroid Build Coastguard Worker        "benchmark_options",
141*dbb99499SAndroid Build Coastguard Worker        metavar="benchmark_options",
142*dbb99499SAndroid Build Coastguard Worker        nargs=argparse.REMAINDER,
143*dbb99499SAndroid Build Coastguard Worker        help="Arguments to pass when running benchmark executables",
144*dbb99499SAndroid Build Coastguard Worker    )
145*dbb99499SAndroid Build Coastguard Worker
146*dbb99499SAndroid Build Coastguard Worker    parser_b = subparsers.add_parser(
147*dbb99499SAndroid Build Coastguard Worker        "filters", help="Compare filter one with the filter two of benchmark"
148*dbb99499SAndroid Build Coastguard Worker    )
149*dbb99499SAndroid Build Coastguard Worker    baseline = parser_b.add_argument_group("baseline", "The benchmark baseline")
150*dbb99499SAndroid Build Coastguard Worker    baseline.add_argument(
151*dbb99499SAndroid Build Coastguard Worker        "test",
152*dbb99499SAndroid Build Coastguard Worker        metavar="test",
153*dbb99499SAndroid Build Coastguard Worker        type=argparse.FileType("r"),
154*dbb99499SAndroid Build Coastguard Worker        nargs=1,
155*dbb99499SAndroid Build Coastguard Worker        help="A benchmark executable or JSON output file",
156*dbb99499SAndroid Build Coastguard Worker    )
157*dbb99499SAndroid Build Coastguard Worker    baseline.add_argument(
158*dbb99499SAndroid Build Coastguard Worker        "filter_baseline",
159*dbb99499SAndroid Build Coastguard Worker        metavar="filter_baseline",
160*dbb99499SAndroid Build Coastguard Worker        type=str,
161*dbb99499SAndroid Build Coastguard Worker        nargs=1,
162*dbb99499SAndroid Build Coastguard Worker        help="The first filter, that will be used as baseline",
163*dbb99499SAndroid Build Coastguard Worker    )
164*dbb99499SAndroid Build Coastguard Worker    contender = parser_b.add_argument_group(
165*dbb99499SAndroid Build Coastguard Worker        "contender", "The benchmark that will be compared against the baseline"
166*dbb99499SAndroid Build Coastguard Worker    )
167*dbb99499SAndroid Build Coastguard Worker    contender.add_argument(
168*dbb99499SAndroid Build Coastguard Worker        "filter_contender",
169*dbb99499SAndroid Build Coastguard Worker        metavar="filter_contender",
170*dbb99499SAndroid Build Coastguard Worker        type=str,
171*dbb99499SAndroid Build Coastguard Worker        nargs=1,
172*dbb99499SAndroid Build Coastguard Worker        help="The second filter, that will be compared against the baseline",
173*dbb99499SAndroid Build Coastguard Worker    )
174*dbb99499SAndroid Build Coastguard Worker    parser_b.add_argument(
175*dbb99499SAndroid Build Coastguard Worker        "benchmark_options",
176*dbb99499SAndroid Build Coastguard Worker        metavar="benchmark_options",
177*dbb99499SAndroid Build Coastguard Worker        nargs=argparse.REMAINDER,
178*dbb99499SAndroid Build Coastguard Worker        help="Arguments to pass when running benchmark executables",
179*dbb99499SAndroid Build Coastguard Worker    )
180*dbb99499SAndroid Build Coastguard Worker
181*dbb99499SAndroid Build Coastguard Worker    parser_c = subparsers.add_parser(
182*dbb99499SAndroid Build Coastguard Worker        "benchmarksfiltered",
183*dbb99499SAndroid Build Coastguard Worker        help="Compare filter one of first benchmark with filter two of the second benchmark",
184*dbb99499SAndroid Build Coastguard Worker    )
185*dbb99499SAndroid Build Coastguard Worker    baseline = parser_c.add_argument_group("baseline", "The benchmark baseline")
186*dbb99499SAndroid Build Coastguard Worker    baseline.add_argument(
187*dbb99499SAndroid Build Coastguard Worker        "test_baseline",
188*dbb99499SAndroid Build Coastguard Worker        metavar="test_baseline",
189*dbb99499SAndroid Build Coastguard Worker        type=argparse.FileType("r"),
190*dbb99499SAndroid Build Coastguard Worker        nargs=1,
191*dbb99499SAndroid Build Coastguard Worker        help="A benchmark executable or JSON output file",
192*dbb99499SAndroid Build Coastguard Worker    )
193*dbb99499SAndroid Build Coastguard Worker    baseline.add_argument(
194*dbb99499SAndroid Build Coastguard Worker        "filter_baseline",
195*dbb99499SAndroid Build Coastguard Worker        metavar="filter_baseline",
196*dbb99499SAndroid Build Coastguard Worker        type=str,
197*dbb99499SAndroid Build Coastguard Worker        nargs=1,
198*dbb99499SAndroid Build Coastguard Worker        help="The first filter, that will be used as baseline",
199*dbb99499SAndroid Build Coastguard Worker    )
200*dbb99499SAndroid Build Coastguard Worker    contender = parser_c.add_argument_group(
201*dbb99499SAndroid Build Coastguard Worker        "contender", "The benchmark that will be compared against the baseline"
202*dbb99499SAndroid Build Coastguard Worker    )
203*dbb99499SAndroid Build Coastguard Worker    contender.add_argument(
204*dbb99499SAndroid Build Coastguard Worker        "test_contender",
205*dbb99499SAndroid Build Coastguard Worker        metavar="test_contender",
206*dbb99499SAndroid Build Coastguard Worker        type=argparse.FileType("r"),
207*dbb99499SAndroid Build Coastguard Worker        nargs=1,
208*dbb99499SAndroid Build Coastguard Worker        help="The second benchmark executable or JSON output file, that will be compared against the baseline",
209*dbb99499SAndroid Build Coastguard Worker    )
210*dbb99499SAndroid Build Coastguard Worker    contender.add_argument(
211*dbb99499SAndroid Build Coastguard Worker        "filter_contender",
212*dbb99499SAndroid Build Coastguard Worker        metavar="filter_contender",
213*dbb99499SAndroid Build Coastguard Worker        type=str,
214*dbb99499SAndroid Build Coastguard Worker        nargs=1,
215*dbb99499SAndroid Build Coastguard Worker        help="The second filter, that will be compared against the baseline",
216*dbb99499SAndroid Build Coastguard Worker    )
217*dbb99499SAndroid Build Coastguard Worker    parser_c.add_argument(
218*dbb99499SAndroid Build Coastguard Worker        "benchmark_options",
219*dbb99499SAndroid Build Coastguard Worker        metavar="benchmark_options",
220*dbb99499SAndroid Build Coastguard Worker        nargs=argparse.REMAINDER,
221*dbb99499SAndroid Build Coastguard Worker        help="Arguments to pass when running benchmark executables",
222*dbb99499SAndroid Build Coastguard Worker    )
223*dbb99499SAndroid Build Coastguard Worker
224*dbb99499SAndroid Build Coastguard Worker    return parser
225*dbb99499SAndroid Build Coastguard Worker
226*dbb99499SAndroid Build Coastguard Worker
227*dbb99499SAndroid Build Coastguard Workerdef main():
228*dbb99499SAndroid Build Coastguard Worker    # Parse the command line flags
229*dbb99499SAndroid Build Coastguard Worker    parser = create_parser()
230*dbb99499SAndroid Build Coastguard Worker    args, unknown_args = parser.parse_known_args()
231*dbb99499SAndroid Build Coastguard Worker    if args.mode is None:
232*dbb99499SAndroid Build Coastguard Worker        parser.print_help()
233*dbb99499SAndroid Build Coastguard Worker        exit(1)
234*dbb99499SAndroid Build Coastguard Worker    assert not unknown_args
235*dbb99499SAndroid Build Coastguard Worker    benchmark_options = args.benchmark_options
236*dbb99499SAndroid Build Coastguard Worker
237*dbb99499SAndroid Build Coastguard Worker    if args.mode == "benchmarks":
238*dbb99499SAndroid Build Coastguard Worker        test_baseline = args.test_baseline[0].name
239*dbb99499SAndroid Build Coastguard Worker        test_contender = args.test_contender[0].name
240*dbb99499SAndroid Build Coastguard Worker        filter_baseline = ""
241*dbb99499SAndroid Build Coastguard Worker        filter_contender = ""
242*dbb99499SAndroid Build Coastguard Worker
243*dbb99499SAndroid Build Coastguard Worker        # NOTE: if test_baseline == test_contender, you are analyzing the stdev
244*dbb99499SAndroid Build Coastguard Worker
245*dbb99499SAndroid Build Coastguard Worker        description = "Comparing %s to %s" % (test_baseline, test_contender)
246*dbb99499SAndroid Build Coastguard Worker    elif args.mode == "filters":
247*dbb99499SAndroid Build Coastguard Worker        test_baseline = args.test[0].name
248*dbb99499SAndroid Build Coastguard Worker        test_contender = args.test[0].name
249*dbb99499SAndroid Build Coastguard Worker        filter_baseline = args.filter_baseline[0]
250*dbb99499SAndroid Build Coastguard Worker        filter_contender = args.filter_contender[0]
251*dbb99499SAndroid Build Coastguard Worker
252*dbb99499SAndroid Build Coastguard Worker        # NOTE: if filter_baseline == filter_contender, you are analyzing the
253*dbb99499SAndroid Build Coastguard Worker        # stdev
254*dbb99499SAndroid Build Coastguard Worker
255*dbb99499SAndroid Build Coastguard Worker        description = "Comparing %s to %s (from %s)" % (
256*dbb99499SAndroid Build Coastguard Worker            filter_baseline,
257*dbb99499SAndroid Build Coastguard Worker            filter_contender,
258*dbb99499SAndroid Build Coastguard Worker            args.test[0].name,
259*dbb99499SAndroid Build Coastguard Worker        )
260*dbb99499SAndroid Build Coastguard Worker    elif args.mode == "benchmarksfiltered":
261*dbb99499SAndroid Build Coastguard Worker        test_baseline = args.test_baseline[0].name
262*dbb99499SAndroid Build Coastguard Worker        test_contender = args.test_contender[0].name
263*dbb99499SAndroid Build Coastguard Worker        filter_baseline = args.filter_baseline[0]
264*dbb99499SAndroid Build Coastguard Worker        filter_contender = args.filter_contender[0]
265*dbb99499SAndroid Build Coastguard Worker
266*dbb99499SAndroid Build Coastguard Worker        # NOTE: if test_baseline == test_contender and
267*dbb99499SAndroid Build Coastguard Worker        # filter_baseline == filter_contender, you are analyzing the stdev
268*dbb99499SAndroid Build Coastguard Worker
269*dbb99499SAndroid Build Coastguard Worker        description = "Comparing %s (from %s) to %s (from %s)" % (
270*dbb99499SAndroid Build Coastguard Worker            filter_baseline,
271*dbb99499SAndroid Build Coastguard Worker            test_baseline,
272*dbb99499SAndroid Build Coastguard Worker            filter_contender,
273*dbb99499SAndroid Build Coastguard Worker            test_contender,
274*dbb99499SAndroid Build Coastguard Worker        )
275*dbb99499SAndroid Build Coastguard Worker    else:
276*dbb99499SAndroid Build Coastguard Worker        # should never happen
277*dbb99499SAndroid Build Coastguard Worker        print("Unrecognized mode of operation: '%s'" % args.mode)
278*dbb99499SAndroid Build Coastguard Worker        parser.print_help()
279*dbb99499SAndroid Build Coastguard Worker        exit(1)
280*dbb99499SAndroid Build Coastguard Worker
281*dbb99499SAndroid Build Coastguard Worker    check_inputs(test_baseline, test_contender, benchmark_options)
282*dbb99499SAndroid Build Coastguard Worker
283*dbb99499SAndroid Build Coastguard Worker    if args.display_aggregates_only:
284*dbb99499SAndroid Build Coastguard Worker        benchmark_options += ["--benchmark_display_aggregates_only=true"]
285*dbb99499SAndroid Build Coastguard Worker
286*dbb99499SAndroid Build Coastguard Worker    options_baseline = []
287*dbb99499SAndroid Build Coastguard Worker    options_contender = []
288*dbb99499SAndroid Build Coastguard Worker
289*dbb99499SAndroid Build Coastguard Worker    if filter_baseline and filter_contender:
290*dbb99499SAndroid Build Coastguard Worker        options_baseline = ["--benchmark_filter=%s" % filter_baseline]
291*dbb99499SAndroid Build Coastguard Worker        options_contender = ["--benchmark_filter=%s" % filter_contender]
292*dbb99499SAndroid Build Coastguard Worker
293*dbb99499SAndroid Build Coastguard Worker    # Run the benchmarks and report the results
294*dbb99499SAndroid Build Coastguard Worker    json1 = json1_orig = gbench.util.sort_benchmark_results(
295*dbb99499SAndroid Build Coastguard Worker        gbench.util.run_or_load_benchmark(
296*dbb99499SAndroid Build Coastguard Worker            test_baseline, benchmark_options + options_baseline
297*dbb99499SAndroid Build Coastguard Worker        )
298*dbb99499SAndroid Build Coastguard Worker    )
299*dbb99499SAndroid Build Coastguard Worker    json2 = json2_orig = gbench.util.sort_benchmark_results(
300*dbb99499SAndroid Build Coastguard Worker        gbench.util.run_or_load_benchmark(
301*dbb99499SAndroid Build Coastguard Worker            test_contender, benchmark_options + options_contender
302*dbb99499SAndroid Build Coastguard Worker        )
303*dbb99499SAndroid Build Coastguard Worker    )
304*dbb99499SAndroid Build Coastguard Worker
305*dbb99499SAndroid Build Coastguard Worker    # Now, filter the benchmarks so that the difference report can work
306*dbb99499SAndroid Build Coastguard Worker    if filter_baseline and filter_contender:
307*dbb99499SAndroid Build Coastguard Worker        replacement = "[%s vs. %s]" % (filter_baseline, filter_contender)
308*dbb99499SAndroid Build Coastguard Worker        json1 = gbench.report.filter_benchmark(
309*dbb99499SAndroid Build Coastguard Worker            json1_orig, filter_baseline, replacement
310*dbb99499SAndroid Build Coastguard Worker        )
311*dbb99499SAndroid Build Coastguard Worker        json2 = gbench.report.filter_benchmark(
312*dbb99499SAndroid Build Coastguard Worker            json2_orig, filter_contender, replacement
313*dbb99499SAndroid Build Coastguard Worker        )
314*dbb99499SAndroid Build Coastguard Worker
315*dbb99499SAndroid Build Coastguard Worker    diff_report = gbench.report.get_difference_report(json1, json2, args.utest)
316*dbb99499SAndroid Build Coastguard Worker    output_lines = gbench.report.print_difference_report(
317*dbb99499SAndroid Build Coastguard Worker        diff_report,
318*dbb99499SAndroid Build Coastguard Worker        args.display_aggregates_only,
319*dbb99499SAndroid Build Coastguard Worker        args.utest,
320*dbb99499SAndroid Build Coastguard Worker        args.utest_alpha,
321*dbb99499SAndroid Build Coastguard Worker        args.color,
322*dbb99499SAndroid Build Coastguard Worker    )
323*dbb99499SAndroid Build Coastguard Worker    print(description)
324*dbb99499SAndroid Build Coastguard Worker    for ln in output_lines:
325*dbb99499SAndroid Build Coastguard Worker        print(ln)
326*dbb99499SAndroid Build Coastguard Worker
327*dbb99499SAndroid Build Coastguard Worker    # Optionally, diff and output to JSON
328*dbb99499SAndroid Build Coastguard Worker    if args.dump_to_json is not None:
329*dbb99499SAndroid Build Coastguard Worker        with open(args.dump_to_json, "w") as f_json:
330*dbb99499SAndroid Build Coastguard Worker            json.dump(diff_report, f_json, indent=1)
331*dbb99499SAndroid Build Coastguard Worker
332*dbb99499SAndroid Build Coastguard Worker
333*dbb99499SAndroid Build Coastguard Workerclass TestParser(unittest.TestCase):
334*dbb99499SAndroid Build Coastguard Worker    def setUp(self):
335*dbb99499SAndroid Build Coastguard Worker        self.parser = create_parser()
336*dbb99499SAndroid Build Coastguard Worker        testInputs = os.path.join(
337*dbb99499SAndroid Build Coastguard Worker            os.path.dirname(os.path.realpath(__file__)), "gbench", "Inputs"
338*dbb99499SAndroid Build Coastguard Worker        )
339*dbb99499SAndroid Build Coastguard Worker        self.testInput0 = os.path.join(testInputs, "test1_run1.json")
340*dbb99499SAndroid Build Coastguard Worker        self.testInput1 = os.path.join(testInputs, "test1_run2.json")
341*dbb99499SAndroid Build Coastguard Worker
342*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_basic(self):
343*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
344*dbb99499SAndroid Build Coastguard Worker            ["benchmarks", self.testInput0, self.testInput1]
345*dbb99499SAndroid Build Coastguard Worker        )
346*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
347*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
348*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
349*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
350*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
351*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
352*dbb99499SAndroid Build Coastguard Worker
353*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_basic_without_utest(self):
354*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
355*dbb99499SAndroid Build Coastguard Worker            ["--no-utest", "benchmarks", self.testInput0, self.testInput1]
356*dbb99499SAndroid Build Coastguard Worker        )
357*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
358*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.utest)
359*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.utest_alpha, 0.05)
360*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
361*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
362*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
363*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
364*dbb99499SAndroid Build Coastguard Worker
365*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_basic_display_aggregates_only(self):
366*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
367*dbb99499SAndroid Build Coastguard Worker            ["-a", "benchmarks", self.testInput0, self.testInput1]
368*dbb99499SAndroid Build Coastguard Worker        )
369*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.display_aggregates_only)
370*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
371*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
372*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
373*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
374*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
375*dbb99499SAndroid Build Coastguard Worker
376*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_basic_with_utest_alpha(self):
377*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
378*dbb99499SAndroid Build Coastguard Worker            ["--alpha=0.314", "benchmarks", self.testInput0, self.testInput1]
379*dbb99499SAndroid Build Coastguard Worker        )
380*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
381*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
382*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.utest_alpha, 0.314)
383*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
384*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
385*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
386*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
387*dbb99499SAndroid Build Coastguard Worker
388*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_basic_without_utest_with_utest_alpha(self):
389*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
390*dbb99499SAndroid Build Coastguard Worker            [
391*dbb99499SAndroid Build Coastguard Worker                "--no-utest",
392*dbb99499SAndroid Build Coastguard Worker                "--alpha=0.314",
393*dbb99499SAndroid Build Coastguard Worker                "benchmarks",
394*dbb99499SAndroid Build Coastguard Worker                self.testInput0,
395*dbb99499SAndroid Build Coastguard Worker                self.testInput1,
396*dbb99499SAndroid Build Coastguard Worker            ]
397*dbb99499SAndroid Build Coastguard Worker        )
398*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
399*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.utest)
400*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.utest_alpha, 0.314)
401*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
402*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
403*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
404*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
405*dbb99499SAndroid Build Coastguard Worker
406*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_with_remainder(self):
407*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
408*dbb99499SAndroid Build Coastguard Worker            ["benchmarks", self.testInput0, self.testInput1, "d"]
409*dbb99499SAndroid Build Coastguard Worker        )
410*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
411*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
412*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
413*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
414*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
415*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options, ["d"])
416*dbb99499SAndroid Build Coastguard Worker
417*dbb99499SAndroid Build Coastguard Worker    def test_benchmarks_with_remainder_after_doubleminus(self):
418*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
419*dbb99499SAndroid Build Coastguard Worker            ["benchmarks", self.testInput0, self.testInput1, "--", "e"]
420*dbb99499SAndroid Build Coastguard Worker        )
421*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
422*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
423*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarks")
424*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
425*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
426*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options, ["e"])
427*dbb99499SAndroid Build Coastguard Worker
428*dbb99499SAndroid Build Coastguard Worker    def test_filters_basic(self):
429*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(["filters", self.testInput0, "c", "d"])
430*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
431*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
432*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "filters")
433*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test[0].name, self.testInput0)
434*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
435*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "d")
436*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
437*dbb99499SAndroid Build Coastguard Worker
438*dbb99499SAndroid Build Coastguard Worker    def test_filters_with_remainder(self):
439*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
440*dbb99499SAndroid Build Coastguard Worker            ["filters", self.testInput0, "c", "d", "e"]
441*dbb99499SAndroid Build Coastguard Worker        )
442*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
443*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
444*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "filters")
445*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test[0].name, self.testInput0)
446*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
447*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "d")
448*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options, ["e"])
449*dbb99499SAndroid Build Coastguard Worker
450*dbb99499SAndroid Build Coastguard Worker    def test_filters_with_remainder_after_doubleminus(self):
451*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
452*dbb99499SAndroid Build Coastguard Worker            ["filters", self.testInput0, "c", "d", "--", "f"]
453*dbb99499SAndroid Build Coastguard Worker        )
454*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
455*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
456*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "filters")
457*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test[0].name, self.testInput0)
458*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
459*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "d")
460*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options, ["f"])
461*dbb99499SAndroid Build Coastguard Worker
462*dbb99499SAndroid Build Coastguard Worker    def test_benchmarksfiltered_basic(self):
463*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
464*dbb99499SAndroid Build Coastguard Worker            ["benchmarksfiltered", self.testInput0, "c", self.testInput1, "e"]
465*dbb99499SAndroid Build Coastguard Worker        )
466*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
467*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
468*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarksfiltered")
469*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
470*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
471*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
472*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "e")
473*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.benchmark_options)
474*dbb99499SAndroid Build Coastguard Worker
475*dbb99499SAndroid Build Coastguard Worker    def test_benchmarksfiltered_with_remainder(self):
476*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
477*dbb99499SAndroid Build Coastguard Worker            [
478*dbb99499SAndroid Build Coastguard Worker                "benchmarksfiltered",
479*dbb99499SAndroid Build Coastguard Worker                self.testInput0,
480*dbb99499SAndroid Build Coastguard Worker                "c",
481*dbb99499SAndroid Build Coastguard Worker                self.testInput1,
482*dbb99499SAndroid Build Coastguard Worker                "e",
483*dbb99499SAndroid Build Coastguard Worker                "f",
484*dbb99499SAndroid Build Coastguard Worker            ]
485*dbb99499SAndroid Build Coastguard Worker        )
486*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
487*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
488*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarksfiltered")
489*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
490*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
491*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
492*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "e")
493*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options[0], "f")
494*dbb99499SAndroid Build Coastguard Worker
495*dbb99499SAndroid Build Coastguard Worker    def test_benchmarksfiltered_with_remainder_after_doubleminus(self):
496*dbb99499SAndroid Build Coastguard Worker        parsed = self.parser.parse_args(
497*dbb99499SAndroid Build Coastguard Worker            [
498*dbb99499SAndroid Build Coastguard Worker                "benchmarksfiltered",
499*dbb99499SAndroid Build Coastguard Worker                self.testInput0,
500*dbb99499SAndroid Build Coastguard Worker                "c",
501*dbb99499SAndroid Build Coastguard Worker                self.testInput1,
502*dbb99499SAndroid Build Coastguard Worker                "e",
503*dbb99499SAndroid Build Coastguard Worker                "--",
504*dbb99499SAndroid Build Coastguard Worker                "g",
505*dbb99499SAndroid Build Coastguard Worker            ]
506*dbb99499SAndroid Build Coastguard Worker        )
507*dbb99499SAndroid Build Coastguard Worker        self.assertFalse(parsed.display_aggregates_only)
508*dbb99499SAndroid Build Coastguard Worker        self.assertTrue(parsed.utest)
509*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.mode, "benchmarksfiltered")
510*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_baseline[0].name, self.testInput0)
511*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_baseline[0], "c")
512*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.test_contender[0].name, self.testInput1)
513*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.filter_contender[0], "e")
514*dbb99499SAndroid Build Coastguard Worker        self.assertEqual(parsed.benchmark_options[0], "g")
515*dbb99499SAndroid Build Coastguard Worker
516*dbb99499SAndroid Build Coastguard Worker
517*dbb99499SAndroid Build Coastguard Workerif __name__ == "__main__":
518*dbb99499SAndroid Build Coastguard Worker    # unittest.main()
519*dbb99499SAndroid Build Coastguard Worker    main()
520*dbb99499SAndroid Build Coastguard Worker
521*dbb99499SAndroid Build Coastguard Worker# vim: tabstop=4 expandtab shiftwidth=4 softtabstop=4
522*dbb99499SAndroid Build Coastguard Worker# kate: tab-width: 4; replace-tabs on; indent-width 4; tab-indents: off;
523*dbb99499SAndroid Build Coastguard Worker# kate: indent-mode python; remove-trailing-spaces modified;
524