xref: /aosp_15_r20/external/cronet/testing/scripts/rust/test_results.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b)
1*6777b538SAndroid Build Coastguard Worker# Copyright 2021 The Chromium Authors
2*6777b538SAndroid Build Coastguard Worker# Use of this source code is governed by a BSD-style license that can be
3*6777b538SAndroid Build Coastguard Worker# found in the LICENSE file.
4*6777b538SAndroid Build Coastguard Worker"""This is a library for printing test result as specified by
5*6777b538SAndroid Build Coastguard Worker//docs/testing/test_executable_api.md (e.g. --isolated-script-test-output)
6*6777b538SAndroid Build Coastguard Workerand //docs/testing/json_test_results_format.md
7*6777b538SAndroid Build Coastguard Worker
8*6777b538SAndroid Build Coastguard WorkerTypical usage:
9*6777b538SAndroid Build Coastguard Worker    import argparse
10*6777b538SAndroid Build Coastguard Worker    import test_results
11*6777b538SAndroid Build Coastguard Worker
12*6777b538SAndroid Build Coastguard Worker    cmdline_parser = argparse.ArgumentParser()
13*6777b538SAndroid Build Coastguard Worker    test_results.add_cmdline_args(cmdline_parser)
14*6777b538SAndroid Build Coastguard Worker    ... adding other cmdline parameter definitions ...
15*6777b538SAndroid Build Coastguard Worker    parsed_cmdline_args = cmdline_parser.parse_args()
16*6777b538SAndroid Build Coastguard Worker
17*6777b538SAndroid Build Coastguard Worker    test_results = []
18*6777b538SAndroid Build Coastguard Worker    test_results.append(test_results.TestResult(
19*6777b538SAndroid Build Coastguard Worker        'test-suite/test-name', 'PASS'))
20*6777b538SAndroid Build Coastguard Worker    ...
21*6777b538SAndroid Build Coastguard Worker
22*6777b538SAndroid Build Coastguard Worker    test_results.print_test_results(parsed_cmdline_args, test_results)
23*6777b538SAndroid Build Coastguard Worker"""
24*6777b538SAndroid Build Coastguard Worker
25*6777b538SAndroid Build Coastguard Workerimport argparse
26*6777b538SAndroid Build Coastguard Workerimport json
27*6777b538SAndroid Build Coastguard Workerimport os
28*6777b538SAndroid Build Coastguard Worker
29*6777b538SAndroid Build Coastguard Worker
30*6777b538SAndroid Build Coastguard Workerclass TestResult:
31*6777b538SAndroid Build Coastguard Worker    """TestResult represents a result of executing a single test once.
32*6777b538SAndroid Build Coastguard Worker    """
33*6777b538SAndroid Build Coastguard Worker
34*6777b538SAndroid Build Coastguard Worker    def __init__(self,
35*6777b538SAndroid Build Coastguard Worker                 test_name,
36*6777b538SAndroid Build Coastguard Worker                 actual_test_result,
37*6777b538SAndroid Build Coastguard Worker                 expected_test_result='PASS'):
38*6777b538SAndroid Build Coastguard Worker        self.test_name = test_name
39*6777b538SAndroid Build Coastguard Worker        self.actual_test_result = actual_test_result
40*6777b538SAndroid Build Coastguard Worker        self.expected_test_result = expected_test_result
41*6777b538SAndroid Build Coastguard Worker
42*6777b538SAndroid Build Coastguard Worker    def __eq__(self, other):
43*6777b538SAndroid Build Coastguard Worker        self_tuple = tuple(sorted(self.__dict__.items()))
44*6777b538SAndroid Build Coastguard Worker        other_tuple = tuple(sorted(other.__dict__.items()))
45*6777b538SAndroid Build Coastguard Worker        return self_tuple == other_tuple
46*6777b538SAndroid Build Coastguard Worker
47*6777b538SAndroid Build Coastguard Worker    def __hash__(self):
48*6777b538SAndroid Build Coastguard Worker        return hash(tuple(sorted(self.__dict__.items())))
49*6777b538SAndroid Build Coastguard Worker
50*6777b538SAndroid Build Coastguard Worker    def __repr__(self):
51*6777b538SAndroid Build Coastguard Worker        result = 'TestResult[{}: {}'.format(self.test_name,
52*6777b538SAndroid Build Coastguard Worker                                            self.actual_test_result)
53*6777b538SAndroid Build Coastguard Worker        if self.expected_test_result != 'PASS':
54*6777b538SAndroid Build Coastguard Worker            result += ' (expecting: {})]'.format(self.expected_test_result)
55*6777b538SAndroid Build Coastguard Worker        else:
56*6777b538SAndroid Build Coastguard Worker            result += ']'
57*6777b538SAndroid Build Coastguard Worker        return result
58*6777b538SAndroid Build Coastguard Worker
59*6777b538SAndroid Build Coastguard Worker
60*6777b538SAndroid Build Coastguard Workerdef _validate_output_directory(outdir):
61*6777b538SAndroid Build Coastguard Worker    if not os.path.isdir(outdir):
62*6777b538SAndroid Build Coastguard Worker        raise argparse.ArgumentTypeError('No such directory: ' + outdir)
63*6777b538SAndroid Build Coastguard Worker    return outdir
64*6777b538SAndroid Build Coastguard Worker
65*6777b538SAndroid Build Coastguard Worker
66*6777b538SAndroid Build Coastguard Workerdef add_cmdline_args(argparse_parser):
67*6777b538SAndroid Build Coastguard Worker    """Adds test-result-specific cmdline parameter definitions to
68*6777b538SAndroid Build Coastguard Worker    `argparse_parser`.
69*6777b538SAndroid Build Coastguard Worker
70*6777b538SAndroid Build Coastguard Worker    Args:
71*6777b538SAndroid Build Coastguard Worker        argparse_parser: An object of argparse.ArgumentParser type.
72*6777b538SAndroid Build Coastguard Worker    """
73*6777b538SAndroid Build Coastguard Worker    outdir_help = 'Directory where test results will be written into.'
74*6777b538SAndroid Build Coastguard Worker    argparse_parser.add_argument('--isolated-outdir',
75*6777b538SAndroid Build Coastguard Worker                                 dest='outdir',
76*6777b538SAndroid Build Coastguard Worker                                 help=outdir_help,
77*6777b538SAndroid Build Coastguard Worker                                 metavar='DIRPATH',
78*6777b538SAndroid Build Coastguard Worker                                 type=_validate_output_directory)
79*6777b538SAndroid Build Coastguard Worker
80*6777b538SAndroid Build Coastguard Worker    outfile_help = 'If this argument is provided, then test results in the ' \
81*6777b538SAndroid Build Coastguard Worker                   'JSON Test Results Format will be written here. See also ' \
82*6777b538SAndroid Build Coastguard Worker                   '//docs/testing/json_test_results_format.md'
83*6777b538SAndroid Build Coastguard Worker    argparse_parser.add_argument('--isolated-script-test-output',
84*6777b538SAndroid Build Coastguard Worker                                 dest='test_output',
85*6777b538SAndroid Build Coastguard Worker                                 default=None,
86*6777b538SAndroid Build Coastguard Worker                                 help=outfile_help,
87*6777b538SAndroid Build Coastguard Worker                                 metavar='FILENAME')
88*6777b538SAndroid Build Coastguard Worker
89*6777b538SAndroid Build Coastguard Worker    argparse_parser.add_argument('--isolated-script-test-perf-output',
90*6777b538SAndroid Build Coastguard Worker                                 dest='ignored_perf_output',
91*6777b538SAndroid Build Coastguard Worker                                 default=None,
92*6777b538SAndroid Build Coastguard Worker                                 help='Deprecated and ignored.',
93*6777b538SAndroid Build Coastguard Worker                                 metavar='IGNORED')
94*6777b538SAndroid Build Coastguard Worker
95*6777b538SAndroid Build Coastguard Worker
96*6777b538SAndroid Build Coastguard Workerdef _build_json_data(list_of_test_results, seconds_since_epoch):
97*6777b538SAndroid Build Coastguard Worker    num_failures_by_type = {}
98*6777b538SAndroid Build Coastguard Worker    tests = {}
99*6777b538SAndroid Build Coastguard Worker    for res in list_of_test_results:
100*6777b538SAndroid Build Coastguard Worker        old_count = num_failures_by_type.get(res.actual_test_result, 0)
101*6777b538SAndroid Build Coastguard Worker        num_failures_by_type[res.actual_test_result] = old_count + 1
102*6777b538SAndroid Build Coastguard Worker
103*6777b538SAndroid Build Coastguard Worker        path = res.test_name.split('//')
104*6777b538SAndroid Build Coastguard Worker        group = tests
105*6777b538SAndroid Build Coastguard Worker        for group_name in path[:-1]:
106*6777b538SAndroid Build Coastguard Worker            if not group_name in group:
107*6777b538SAndroid Build Coastguard Worker                group[group_name] = {}
108*6777b538SAndroid Build Coastguard Worker            group = group[group_name]
109*6777b538SAndroid Build Coastguard Worker
110*6777b538SAndroid Build Coastguard Worker        group[path[-1]] = {
111*6777b538SAndroid Build Coastguard Worker            'expected': res.expected_test_result,
112*6777b538SAndroid Build Coastguard Worker            'actual': res.actual_test_result,
113*6777b538SAndroid Build Coastguard Worker        }
114*6777b538SAndroid Build Coastguard Worker
115*6777b538SAndroid Build Coastguard Worker    return {
116*6777b538SAndroid Build Coastguard Worker        'interrupted': False,
117*6777b538SAndroid Build Coastguard Worker        'path_delimiter': '//',
118*6777b538SAndroid Build Coastguard Worker        'seconds_since_epoch': seconds_since_epoch,
119*6777b538SAndroid Build Coastguard Worker        'version': 3,
120*6777b538SAndroid Build Coastguard Worker        'tests': tests,
121*6777b538SAndroid Build Coastguard Worker        'num_failures_by_type': num_failures_by_type,
122*6777b538SAndroid Build Coastguard Worker    }
123*6777b538SAndroid Build Coastguard Worker
124*6777b538SAndroid Build Coastguard Worker
125*6777b538SAndroid Build Coastguard Workerdef print_test_results(argparse_parsed_args, list_of_test_results,
126*6777b538SAndroid Build Coastguard Worker                       testing_start_time_as_seconds_since_epoch):
127*6777b538SAndroid Build Coastguard Worker    """Prints `list_of_test_results` to a file specified on the cmdline.
128*6777b538SAndroid Build Coastguard Worker
129*6777b538SAndroid Build Coastguard Worker    Args:
130*6777b538SAndroid Build Coastguard Worker        argparse_parsed_arg: A result of an earlier call to
131*6777b538SAndroid Build Coastguard Worker          argparse_parser.parse_args() call (where `argparse_parser` has been
132*6777b538SAndroid Build Coastguard Worker          populated via an even earlier call to add_cmdline_args).
133*6777b538SAndroid Build Coastguard Worker        list_of_test_results: A list of TestResult objects.
134*6777b538SAndroid Build Coastguard Worker        testing_start_time_as_seconds_since_epoch: A number from an earlier
135*6777b538SAndroid Build Coastguard Worker          `time.time()` call.
136*6777b538SAndroid Build Coastguard Worker    """
137*6777b538SAndroid Build Coastguard Worker    if argparse_parsed_args.test_output is None:
138*6777b538SAndroid Build Coastguard Worker        return
139*6777b538SAndroid Build Coastguard Worker
140*6777b538SAndroid Build Coastguard Worker    json_data = _build_json_data(list_of_test_results,
141*6777b538SAndroid Build Coastguard Worker                                 testing_start_time_as_seconds_since_epoch)
142*6777b538SAndroid Build Coastguard Worker
143*6777b538SAndroid Build Coastguard Worker    filepath = argparse_parsed_args.test_output
144*6777b538SAndroid Build Coastguard Worker    with open(filepath, mode='w', encoding='utf-8') as f:
145*6777b538SAndroid Build Coastguard Worker        json.dump(json_data, f, indent=2)
146