xref: /aosp_15_r20/external/angle/build/android/pylib/results/json_results.py (revision 8975f5c5ed3d1c378011245431ada316dfb6f244) 
1# Copyright 2014 The Chromium Authors
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6import collections
7import itertools
8import json
9import logging
10import time
11
12from pylib.base import base_test_result
13
14def GenerateResultsDict(test_run_results, global_tags=None):
15  """Create a results dict from |test_run_results| suitable for writing to JSON.
16  Args:
17    test_run_results: a list of base_test_result.TestRunResults objects.
18  Returns:
19    A results dict that mirrors the one generated by
20      base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.
21  """
22  # Example json output.
23  # {
24  #   "global_tags": [],
25  #   "all_tests": [
26  #     "test1",
27  #     "test2",
28  #    ],
29  #   "disabled_tests": [],
30  #   "per_iteration_data": [
31  #     {
32  #       "test1": [
33  #         {
34  #           "status": "SUCCESS",
35  #           "elapsed_time_ms": 1,
36  #           "output_snippet": "",
37  #           "output_snippet_base64": "",
38  #           "losless_snippet": "",
39  #         },
40  #         ...
41  #       ],
42  #       "test2": [
43  #         {
44  #           "status": "FAILURE",
45  #           "elapsed_time_ms": 12,
46  #           "output_snippet": "",
47  #           "output_snippet_base64": "",
48  #           "losless_snippet": "",
49  #         },
50  #         ...
51  #       ],
52  #     },
53  #     {
54  #       "test1": [
55  #         {
56  #           "status": "SUCCESS",
57  #           "elapsed_time_ms": 1,
58  #           "output_snippet": "",
59  #           "output_snippet_base64": "",
60  #           "losless_snippet": "",
61  #         },
62  #       ],
63  #       "test2": [
64  #         {
65  #           "status": "FAILURE",
66  #           "elapsed_time_ms": 12,
67  #           "output_snippet": "",
68  #           "output_snippet_base64": "",
69  #           "losless_snippet": "",
70  #         },
71  #       ],
72  #     },
73  #     ...
74  #   ],
75  # }
76
77  all_tests = set()
78  per_iteration_data = []
79  test_run_links = {}
80
81  for test_run_result in test_run_results:
82    iteration_data = collections.defaultdict(list)
83    if isinstance(test_run_result, list):
84      results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
85      for tr in test_run_result:
86        test_run_links.update(tr.GetLinks())
87
88    else:
89      results_iterable = test_run_result.GetAll()
90      test_run_links.update(test_run_result.GetLinks())
91
92    for r in results_iterable:
93      result_dict = {
94          'status': r.GetType(),
95          'elapsed_time_ms': r.GetDuration(),
96          'output_snippet': r.GetLog(),
97          'losless_snippet': True,
98          'output_snippet_base64': '',
99          'links': r.GetLinks(),
100      }
101      iteration_data[r.GetName()].append(result_dict)
102
103    all_tests = all_tests.union(set(iteration_data.keys()))
104    per_iteration_data.append(iteration_data)
105
106  return {
107    'global_tags': global_tags or [],
108    'all_tests': sorted(list(all_tests)),
109    # TODO(jbudorick): Add support for disabled tests within base_test_result.
110    'disabled_tests': [],
111    'per_iteration_data': per_iteration_data,
112    'links': test_run_links,
113  }
114
115
116def GenerateJsonTestResultFormatDict(test_run_results, interrupted):
117  """Create a results dict from |test_run_results| suitable for writing to JSON.
118
119  Args:
120    test_run_results: a list of base_test_result.TestRunResults objects.
121    interrupted: True if tests were interrupted, e.g. timeout listing tests
122  Returns:
123    A results dict that mirrors the standard JSON Test Results Format.
124  """
125
126  tests = {}
127  counts = {'PASS': 0, 'FAIL': 0, 'SKIP': 0, 'CRASH': 0, 'TIMEOUT': 0}
128
129  for test_run_result in test_run_results:
130    if isinstance(test_run_result, list):
131      results_iterable = itertools.chain(*(t.GetAll() for t in test_run_result))
132    else:
133      results_iterable = test_run_result.GetAll()
134
135    for r in results_iterable:
136      element = tests
137      for key in r.GetName().split('.'):
138        if key not in element:
139          element[key] = {}
140        element = element[key]
141
142      element['expected'] = 'PASS'
143
144      if r.GetType() == base_test_result.ResultType.PASS:
145        result = 'PASS'
146      elif r.GetType() == base_test_result.ResultType.SKIP:
147        result = 'SKIP'
148      elif r.GetType() == base_test_result.ResultType.CRASH:
149        result = 'CRASH'
150      elif r.GetType() == base_test_result.ResultType.TIMEOUT:
151        result = 'TIMEOUT'
152      else:
153        result = 'FAIL'
154
155      if 'actual' in element:
156        element['actual'] += ' ' + result
157      else:
158        counts[result] += 1
159        element['actual'] = result
160        if result == 'FAIL':
161          element['is_unexpected'] = True
162
163      if r.GetDuration() != 0:
164        element['time'] = r.GetDuration()
165
166  # Fill in required fields.
167  return {
168      'interrupted': interrupted,
169      'num_failures_by_type': counts,
170      'path_delimiter': '.',
171      'seconds_since_epoch': time.time(),
172      'tests': tests,
173      'version': 3,
174  }
175
176
177def GenerateJsonResultsFile(test_run_result, file_path, global_tags=None,
178                            **kwargs):
179  """Write |test_run_result| to JSON.
180
181  This emulates the format of the JSON emitted by
182  base/test/launcher/test_results_tracker.cc:SaveSummaryAsJSON.
183
184  Args:
185    test_run_result: a base_test_result.TestRunResults object.
186    file_path: The path to the JSON file to write.
187  """
188  with open(file_path, 'w') as json_result_file:
189    json_result_file.write(json.dumps(
190        GenerateResultsDict(test_run_result, global_tags=global_tags),
191        **kwargs))
192    logging.info('Generated json results file at %s', file_path)
193
194
195def GenerateJsonTestResultFormatFile(test_run_result, interrupted, file_path,
196                                     **kwargs):
197  """Write |test_run_result| to JSON.
198
199  This uses the official Chromium Test Results Format.
200
201  Args:
202    test_run_result: a base_test_result.TestRunResults object.
203    interrupted: True if tests were interrupted, e.g. timeout listing tests
204    file_path: The path to the JSON file to write.
205  """
206  with open(file_path, 'w') as json_result_file:
207    json_result_file.write(
208        json.dumps(
209            GenerateJsonTestResultFormatDict(test_run_result, interrupted),
210            **kwargs))
211    logging.info('Generated json results file at %s', file_path)
212
213
214def ParseResultsFromJson(json_results):
215  """Creates a list of BaseTestResult objects from JSON.
216
217  Args:
218    json_results: A JSON dict in the format created by
219                  GenerateJsonResultsFile.
220  """
221
222  def string_as_status(s):
223    if s in base_test_result.ResultType.GetTypes():
224      return s
225    return base_test_result.ResultType.UNKNOWN
226
227  results_list = []
228  testsuite_runs = json_results['per_iteration_data']
229  for testsuite_run in testsuite_runs:
230    for test, test_runs in testsuite_run.items():
231      results_list.extend(
232          [base_test_result.BaseTestResult(test,
233                                           string_as_status(tr['status']),
234                                           duration=tr['elapsed_time_ms'],
235                                           log=tr.get('output_snippet'))
236          for tr in test_runs])
237  return results_list
238