1#!/usr/bin/env python 2# Copyright 2016 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import copy 9import json 10import sys 11 12# These fields must appear in the test result output 13REQUIRED = { 14 'interrupted', 15 'num_failures_by_type', 16 'seconds_since_epoch', 17 'tests', 18 } 19 20# These fields are optional, but must have the same value on all shards 21OPTIONAL_MATCHING = ( 22 'builder_name', 23 'build_number', 24 'chromium_revision', 25 'has_pretty_patch', 26 'has_wdiff', 27 'path_delimiter', 28 'pixel_tests_enabled', 29 'random_order_seed' 30 ) 31 32# The last shard's value for these fields will show up in the merged results 33OPTIONAL_IGNORED = ( 34 'layout_tests_dir', 35 'metadata' 36 ) 37 38# These fields are optional and will be summed together 39OPTIONAL_COUNTS = ( 40 'fixable', 41 'num_flaky', 42 'num_passes', 43 'num_regressions', 44 'skipped', 45 'skips', 46 ) 47 48 49class MergeException(Exception): 50 pass 51 52 53def merge_test_results(shard_results_list): 54 """ Merge list of results. 55 56 Args: 57 shard_results_list: list of results to merge. All the results must have the 58 same format. Supported format are simplified JSON format & Chromium JSON 59 test results format version 3 (see 60 https://www.chromium.org/developers/the-json-test-results-format) 61 62 Returns: 63 a dictionary that represent the merged results. Its format follow the same 64 format of all results in |shard_results_list|. 65 """ 66 shard_results_list = [x for x in shard_results_list if x] 67 if not shard_results_list: 68 return {} 69 70 if 'seconds_since_epoch' in shard_results_list[0]: 71 return _merge_json_test_result_format(shard_results_list) 72 73 return _merge_simplified_json_format(shard_results_list) 74 75 76def _merge_simplified_json_format(shard_results_list): 77 # This code is specialized to the "simplified" JSON format that used to be 78 # the standard for recipes. 79 80 # These are the only keys we pay attention to in the output JSON. 81 merged_results = { 82 'successes': [], 83 'failures': [], 84 'valid': True, 85 } 86 87 for result_json in shard_results_list: 88 successes = result_json.get('successes', []) 89 failures = result_json.get('failures', []) 90 valid = result_json.get('valid', True) 91 92 if (not isinstance(successes, list) or not isinstance(failures, list) or 93 not isinstance(valid, bool)): 94 raise MergeException( 95 'Unexpected value type in %s' % result_json) # pragma: no cover 96 97 merged_results['successes'].extend(successes) 98 merged_results['failures'].extend(failures) 99 merged_results['valid'] = merged_results['valid'] and valid 100 return merged_results 101 102 103def _merge_json_test_result_format(shard_results_list): 104 # This code is specialized to the Chromium JSON test results format version 3: 105 # https://www.chromium.org/developers/the-json-test-results-format 106 107 # These are required fields for the JSON test result format version 3. 108 merged_results = { 109 'tests': {}, 110 'interrupted': False, 111 'version': 3, 112 'seconds_since_epoch': float('inf'), 113 'num_failures_by_type': { 114 } 115 } 116 117 # To make sure that we don't mutate existing shard_results_list. 118 shard_results_list = copy.deepcopy(shard_results_list) 119 for result_json in shard_results_list: 120 # TODO(tansell): check whether this deepcopy is actually necessary. 121 result_json = copy.deepcopy(result_json) 122 123 # Check the version first 124 version = result_json.pop('version', -1) 125 if version != 3: 126 raise MergeException( # pragma: no cover (covered by 127 # results_merger_unittest). 128 'Unsupported version %s. Only version 3 is supported' % version) 129 130 # Check the results for each shard have the required keys 131 missing = REQUIRED - set(result_json) 132 if missing: 133 raise MergeException( # pragma: no cover (covered by 134 # results_merger_unittest). 135 'Invalid json test results (missing %s)' % missing) 136 137 # Curry merge_values for this result_json. 138 merge = lambda key, merge_func: merge_value( 139 result_json, merged_results, key, merge_func) 140 141 # Traverse the result_json's test trie & merged_results's test tries in 142 # DFS order & add the n to merged['tests']. 143 merge('tests', merge_tries) 144 145 # If any were interrupted, we are interrupted. 146 merge('interrupted', lambda x,y: x|y) 147 148 # Use the earliest seconds_since_epoch value 149 merge('seconds_since_epoch', min) 150 151 # Sum the number of failure types 152 merge('num_failures_by_type', sum_dicts) 153 154 # Optional values must match 155 for optional_key in OPTIONAL_MATCHING: 156 if optional_key not in result_json: 157 continue 158 159 if optional_key not in merged_results: 160 # Set this value to None, then blindly copy over it. 161 merged_results[optional_key] = None 162 merge(optional_key, lambda src, dst: src) 163 else: 164 merge(optional_key, ensure_match) 165 166 # Optional values ignored 167 for optional_key in OPTIONAL_IGNORED: 168 if optional_key in result_json: 169 merged_results[optional_key] = result_json.pop( 170 # pragma: no cover (covered by 171 # results_merger_unittest). 172 optional_key) 173 174 # Sum optional value counts 175 for count_key in OPTIONAL_COUNTS: 176 if count_key in result_json: # pragma: no cover 177 # TODO(mcgreevy): add coverage. 178 merged_results.setdefault(count_key, 0) 179 merge(count_key, lambda a, b: a+b) 180 181 if result_json: 182 raise MergeException( # pragma: no cover (covered by 183 # results_merger_unittest). 184 'Unmergable values %s' % list(result_json.keys())) 185 186 return merged_results 187 188 189def merge_tries(source, dest): 190 """ Merges test tries. 191 192 This is intended for use as a merge_func parameter to merge_value. 193 194 Args: 195 source: A result json test trie. 196 dest: A json test trie merge destination. 197 """ 198 # merge_tries merges source into dest by performing a lock-step depth-first 199 # traversal of dest and source. 200 # pending_nodes contains a list of all sub-tries which have been reached but 201 # need further merging. 202 # Each element consists of a trie prefix, and a sub-trie from each of dest 203 # and source which is reached via that prefix. 204 pending_nodes = [('', dest, source)] 205 while pending_nodes: 206 prefix, dest_node, curr_node = pending_nodes.pop() 207 for k, v in curr_node.items(): 208 if k in dest_node: 209 if not isinstance(v, dict): 210 raise MergeException( 211 "%s:%s: %r not mergable, curr_node: %r\ndest_node: %r" % ( 212 prefix, k, v, curr_node, dest_node)) 213 pending_nodes.append(("%s:%s" % (prefix, k), dest_node[k], v)) 214 else: 215 dest_node[k] = v 216 return dest 217 218 219def ensure_match(source, dest): 220 """ Returns source if it matches dest. 221 222 This is intended for use as a merge_func parameter to merge_value. 223 224 Raises: 225 MergeException if source != dest 226 """ 227 if source != dest: 228 raise MergeException( # pragma: no cover (covered by 229 # results_merger_unittest). 230 "Values don't match: %s, %s" % (source, dest)) 231 return source 232 233 234def sum_dicts(source, dest): 235 """ Adds values from source to corresponding values in dest. 236 237 This is intended for use as a merge_func parameter to merge_value. 238 """ 239 for k, v in source.items(): 240 dest.setdefault(k, 0) 241 dest[k] += v 242 243 return dest 244 245 246def merge_value(source, dest, key, merge_func): 247 """ Merges a value from source to dest. 248 249 The value is deleted from source. 250 251 Args: 252 source: A dictionary from which to pull a value, identified by key. 253 dest: The dictionary into to which the value is to be merged. 254 key: The key which identifies the value to be merged. 255 merge_func(src, dst): A function which merges its src into dst, 256 and returns the result. May modify dst. May raise a MergeException. 257 258 Raises: 259 MergeException if the values can not be merged. 260 """ 261 try: 262 dest[key] = merge_func(source[key], dest[key]) 263 except MergeException as e: 264 message = "MergeFailure for %s\n%s" % (key, e.args[0]) 265 e.args = (message,) + e.args[1:] 266 raise 267 del source[key] 268 269 270def main(files): 271 if len(files) < 2: 272 sys.stderr.write("Not enough JSON files to merge.\n") 273 return 1 274 sys.stderr.write('Starting with %s\n' % files[0]) 275 result = json.load(open(files[0])) 276 for f in files[1:]: 277 sys.stderr.write('Merging %s\n' % f) 278 result = merge_test_results([result, json.load(open(f))]) 279 print(json.dumps(result)) 280 return 0 281 282 283if __name__ == "__main__": 284 sys.exit(main(sys.argv[1:])) 285