xref: /aosp_15_r20/external/cronet/testing/merge_scripts/results_merger.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#!/usr/bin/env python
2# Copyright 2016 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from __future__ import print_function
7
8import copy
9import json
10import sys
11
12# These fields must appear in the test result output
13REQUIRED = {
14    'interrupted',
15    'num_failures_by_type',
16    'seconds_since_epoch',
17    'tests',
18    }
19
20# These fields are optional, but must have the same value on all shards
21OPTIONAL_MATCHING = (
22    'builder_name',
23    'build_number',
24    'chromium_revision',
25    'has_pretty_patch',
26    'has_wdiff',
27    'path_delimiter',
28    'pixel_tests_enabled',
29    'random_order_seed'
30    )
31
32# The last shard's value for these fields will show up in the merged results
33OPTIONAL_IGNORED = (
34    'layout_tests_dir',
35    'metadata'
36    )
37
38# These fields are optional and will be summed together
39OPTIONAL_COUNTS = (
40    'fixable',
41    'num_flaky',
42    'num_passes',
43    'num_regressions',
44    'skipped',
45    'skips',
46    )
47
48
49class MergeException(Exception):
50  pass
51
52
53def merge_test_results(shard_results_list):
54  """ Merge list of results.
55
56  Args:
57    shard_results_list: list of results to merge. All the results must have the
58      same format. Supported format are simplified JSON format & Chromium JSON
59      test results format version 3 (see
60      https://www.chromium.org/developers/the-json-test-results-format)
61
62  Returns:
63    a dictionary that represent the merged results. Its format follow the same
64    format of all results in |shard_results_list|.
65  """
66  shard_results_list = [x for x in shard_results_list if x]
67  if not shard_results_list:
68    return {}
69
70  if 'seconds_since_epoch' in shard_results_list[0]:
71    return _merge_json_test_result_format(shard_results_list)
72
73  return _merge_simplified_json_format(shard_results_list)
74
75
76def _merge_simplified_json_format(shard_results_list):
77  # This code is specialized to the "simplified" JSON format that used to be
78  # the standard for recipes.
79
80  # These are the only keys we pay attention to in the output JSON.
81  merged_results = {
82    'successes': [],
83    'failures': [],
84    'valid': True,
85  }
86
87  for result_json in shard_results_list:
88    successes = result_json.get('successes', [])
89    failures = result_json.get('failures', [])
90    valid = result_json.get('valid', True)
91
92    if (not isinstance(successes, list) or not isinstance(failures, list) or
93        not isinstance(valid, bool)):
94      raise MergeException(
95        'Unexpected value type in %s' % result_json)  # pragma: no cover
96
97    merged_results['successes'].extend(successes)
98    merged_results['failures'].extend(failures)
99    merged_results['valid'] = merged_results['valid'] and valid
100  return merged_results
101
102
103def _merge_json_test_result_format(shard_results_list):
104  # This code is specialized to the Chromium JSON test results format version 3:
105  # https://www.chromium.org/developers/the-json-test-results-format
106
107  # These are required fields for the JSON test result format version 3.
108  merged_results = {
109    'tests': {},
110    'interrupted': False,
111    'version': 3,
112    'seconds_since_epoch': float('inf'),
113    'num_failures_by_type': {
114    }
115  }
116
117  # To make sure that we don't mutate existing shard_results_list.
118  shard_results_list = copy.deepcopy(shard_results_list)
119  for result_json in shard_results_list:
120    # TODO(tansell): check whether this deepcopy is actually necessary.
121    result_json = copy.deepcopy(result_json)
122
123    # Check the version first
124    version = result_json.pop('version', -1)
125    if version != 3:
126      raise MergeException(  # pragma: no cover (covered by
127                             # results_merger_unittest).
128          'Unsupported version %s. Only version 3 is supported' % version)
129
130    # Check the results for each shard have the required keys
131    missing = REQUIRED - set(result_json)
132    if missing:
133      raise MergeException(  # pragma: no cover (covered by
134                             # results_merger_unittest).
135          'Invalid json test results (missing %s)' % missing)
136
137    # Curry merge_values for this result_json.
138    merge = lambda key, merge_func: merge_value(
139        result_json, merged_results, key, merge_func)
140
141    # Traverse the result_json's test trie & merged_results's test tries in
142    # DFS order & add the n to merged['tests'].
143    merge('tests', merge_tries)
144
145    # If any were interrupted, we are interrupted.
146    merge('interrupted', lambda x,y: x|y)
147
148    # Use the earliest seconds_since_epoch value
149    merge('seconds_since_epoch', min)
150
151    # Sum the number of failure types
152    merge('num_failures_by_type', sum_dicts)
153
154    # Optional values must match
155    for optional_key in OPTIONAL_MATCHING:
156      if optional_key not in result_json:
157        continue
158
159      if optional_key not in merged_results:
160        # Set this value to None, then blindly copy over it.
161        merged_results[optional_key] = None
162        merge(optional_key, lambda src, dst: src)
163      else:
164        merge(optional_key, ensure_match)
165
166    # Optional values ignored
167    for optional_key in OPTIONAL_IGNORED:
168      if optional_key in result_json:
169        merged_results[optional_key] = result_json.pop(
170            # pragma: no cover (covered by
171            # results_merger_unittest).
172            optional_key)
173
174    # Sum optional value counts
175    for count_key in OPTIONAL_COUNTS:
176      if count_key in result_json:  # pragma: no cover
177        # TODO(mcgreevy): add coverage.
178        merged_results.setdefault(count_key, 0)
179        merge(count_key, lambda a, b: a+b)
180
181    if result_json:
182      raise MergeException(  # pragma: no cover (covered by
183                             # results_merger_unittest).
184          'Unmergable values %s' % list(result_json.keys()))
185
186  return merged_results
187
188
189def merge_tries(source, dest):
190  """ Merges test tries.
191
192  This is intended for use as a merge_func parameter to merge_value.
193
194  Args:
195      source: A result json test trie.
196      dest: A json test trie merge destination.
197  """
198  # merge_tries merges source into dest by performing a lock-step depth-first
199  # traversal of dest and source.
200  # pending_nodes contains a list of all sub-tries which have been reached but
201  # need further merging.
202  # Each element consists of a trie prefix, and a sub-trie from each of dest
203  # and source which is reached via that prefix.
204  pending_nodes = [('', dest, source)]
205  while pending_nodes:
206    prefix, dest_node, curr_node = pending_nodes.pop()
207    for k, v in curr_node.items():
208      if k in dest_node:
209        if not isinstance(v, dict):
210          raise MergeException(
211              "%s:%s: %r not mergable, curr_node: %r\ndest_node: %r" % (
212                  prefix, k, v, curr_node, dest_node))
213        pending_nodes.append(("%s:%s" % (prefix, k), dest_node[k], v))
214      else:
215        dest_node[k] = v
216  return dest
217
218
219def ensure_match(source, dest):
220  """ Returns source if it matches dest.
221
222  This is intended for use as a merge_func parameter to merge_value.
223
224  Raises:
225      MergeException if source != dest
226  """
227  if source != dest:
228    raise MergeException(  # pragma: no cover (covered by
229                           # results_merger_unittest).
230        "Values don't match: %s, %s" % (source, dest))
231  return source
232
233
234def sum_dicts(source, dest):
235  """ Adds values from source to corresponding values in dest.
236
237  This is intended for use as a merge_func parameter to merge_value.
238  """
239  for k, v in source.items():
240    dest.setdefault(k, 0)
241    dest[k] += v
242
243  return dest
244
245
246def merge_value(source, dest, key, merge_func):
247  """ Merges a value from source to dest.
248
249  The value is deleted from source.
250
251  Args:
252    source: A dictionary from which to pull a value, identified by key.
253    dest: The dictionary into to which the value is to be merged.
254    key: The key which identifies the value to be merged.
255    merge_func(src, dst): A function which merges its src into dst,
256        and returns the result. May modify dst. May raise a MergeException.
257
258  Raises:
259    MergeException if the values can not be merged.
260  """
261  try:
262    dest[key] = merge_func(source[key], dest[key])
263  except MergeException as e:
264    message = "MergeFailure for %s\n%s" % (key, e.args[0])
265    e.args = (message,) + e.args[1:]
266    raise
267  del source[key]
268
269
270def main(files):
271  if len(files) < 2:
272    sys.stderr.write("Not enough JSON files to merge.\n")
273    return 1
274  sys.stderr.write('Starting with %s\n' % files[0])
275  result = json.load(open(files[0]))
276  for f in files[1:]:
277    sys.stderr.write('Merging %s\n' % f)
278    result = merge_test_results([result, json.load(open(f))])
279  print(json.dumps(result))
280  return 0
281
282
283if __name__ == "__main__":
284  sys.exit(main(sys.argv[1:]))
285