xref: /aosp_15_r20/external/cronet/testing/merge_scripts/results_merger_test.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#!/usr/bin/env vpython3
2# Copyright 2016 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6import copy
7import os
8import six
9import sys
10import unittest
11
12THIS_DIR = os.path.dirname(os.path.abspath(__file__))
13
14# For results_merger.
15sys.path.insert(0, os.path.join(THIS_DIR, '..', 'resources'))
16import results_merger
17
18
19GOOD_JSON_TEST_RESULT_0 = {
20  'tests': {
21    'car': {
22      'honda': {
23        'expected': 'PASS',
24        'actual': 'PASS'
25      },
26      'toyota': {
27        'expected': 'FAIL',
28        'actual': 'FAIL'
29      }
30    },
31    'computer': {
32      'dell': {
33        'expected': 'PASS',
34        'actual': 'PASS'
35      }
36    },
37  },
38  'interrupted': False,
39  'version': 3,
40  'seconds_since_epoch': 1406662289.76,
41  'num_failures_by_type': {
42     'FAIL': 0,
43     'PASS': 2
44  },
45  'layout_tests_dir': 'abc'
46}
47
48GOOD_JSON_TEST_RESULT_1 = {
49  'tests': {
50    'car': {
51      'tesla': {
52        'expected': 'PASS',
53        'actual': 'PASS'
54      },
55    },
56    'burger': {
57      'mcdonald': {
58        'expected': 'PASS',
59        'actual': 'PASS'
60      }
61    },
62  },
63  'interrupted': False,
64  'version': 3,
65  'seconds_since_epoch': 1406662283.11,
66  'num_failures_by_type': {
67     'FAIL': 0,
68     'PASS': 2
69  },
70  'layout_tests_dir': '123'
71}
72
73GOOD_JSON_TEST_RESULT_2 = {
74  'tests': {
75    'car': {
76      'mercedes': {
77        'expected': 'PASS',
78        'actual': 'FAIL'
79      },
80    },
81    'burger': {
82      'in n out': {
83        'expected': 'PASS',
84        'actual': 'PASS'
85      }
86    },
87  },
88  'interrupted': True,
89  'version': 3,
90  'seconds_since_epoch': 1406662200.01,
91  'num_failures_by_type': {
92     'FAIL': 1,
93     'PASS': 1
94  }
95}
96
97GOOD_JSON_TEST_RESULT_MERGED = {
98  'tests': {
99    'car': {
100      'tesla': {
101        'expected': 'PASS',
102        'actual': 'PASS'
103      },
104      'mercedes': {
105        'expected': 'PASS',
106        'actual': 'FAIL'
107      },
108      'honda': {
109        'expected': 'PASS',
110        'actual': 'PASS'
111      },
112      'toyota': {
113        'expected': 'FAIL',
114        'actual': 'FAIL'
115      }
116    },
117    'computer': {
118      'dell': {
119        'expected': 'PASS',
120        'actual': 'PASS'
121      }
122    },
123    'burger': {
124      'mcdonald': {
125        'expected': 'PASS',
126        'actual': 'PASS'
127      },
128      'in n out': {
129        'expected': 'PASS',
130        'actual': 'PASS'
131      }
132    }
133  },
134  'interrupted': True,
135  'version': 3,
136  'seconds_since_epoch': 1406662200.01,
137  'num_failures_by_type': {
138    'FAIL': 1,
139    'PASS': 5
140  },
141  'layout_tests_dir': '123'
142}
143
144
145def extend(initial, add):
146  out = copy.deepcopy(initial)
147  out.update(add)
148  return out
149
150
151def remove(initial, keys):
152  out = copy.deepcopy(initial)
153  for k in keys:
154    del out[k]
155  return out
156
157
158
159# These unittests are run in PRESUBMIT, but not by recipe_simulation_test, hence
160# to avoid false alert on missing coverage by recipe_simulation_test, we mark
161# these code as no cover.
162class MergingTest(unittest.TestCase):  # pragma: no cover
163  maxDiff = None  # Show full diff if assertion fail
164
165  def test_merge_tries(self):
166    self.assertEqual(
167        {'a': 'A', 'b': {'c': 'C'}},
168        results_merger.merge_tries(
169            {'a': 'A', 'b': {}}, {'b': {'c': 'C'}}))
170
171  def test_merge_tries_unmergable(self):
172    with six.assertRaisesRegex(self, results_merger.MergeException, "a:b"):
173        results_merger.merge_tries(
174            {'a': {'b': 'A'}}, {'a': {'b': 'C'}})
175
176  def test_merge_metadata(self):
177    metadata1 = {'metadata': {'tags': ['foo', 'bar']}}
178    metadata2 = {'metadata': {'tags': ['foo', 'bat']}}
179    merged_results = results_merger.merge_test_results(
180        [extend(GOOD_JSON_TEST_RESULT_0, metadata1),
181         extend(GOOD_JSON_TEST_RESULT_1, metadata2)])
182    self.assertEqual(
183        merged_results['metadata']['tags'], ['foo', 'bat'])
184
185  def test_merge_json_test_results_nop(self):
186    good_json_results = (
187        GOOD_JSON_TEST_RESULT_0,
188        GOOD_JSON_TEST_RESULT_1,
189        GOOD_JSON_TEST_RESULT_2,
190        GOOD_JSON_TEST_RESULT_MERGED)
191    for j in good_json_results:
192      # Clone so we can check the input dictionaries are not modified
193      a = copy.deepcopy(j)
194      self.assertEqual(results_merger.merge_test_results([a]), j)
195      self.assertEqual(a, j)
196
197  def test_merge_json_test_results_invalid_version(self):
198    with self.assertRaises(results_merger.MergeException):
199      results_merger.merge_test_results([
200          extend(GOOD_JSON_TEST_RESULT_0, {'version': 5}),
201          ])
202
203    with self.assertRaises(results_merger.MergeException):
204      results_merger.merge_test_results([
205          GOOD_JSON_TEST_RESULT_0,
206          extend(GOOD_JSON_TEST_RESULT_1, {'version': 5}),
207          ])
208
209  def test_merge_json_test_results_missing_version(self):
210    with self.assertRaises(results_merger.MergeException):
211      results_merger.merge_test_results([
212          remove(GOOD_JSON_TEST_RESULT_0, ['version']),
213          ])
214
215    with self.assertRaises(results_merger.MergeException):
216      results_merger.merge_test_results([
217          GOOD_JSON_TEST_RESULT_0,
218          remove(GOOD_JSON_TEST_RESULT_1, ['version']),
219          ])
220
221  def test_merge_json_test_results_invalid_extra(self):
222    with self.assertRaises(results_merger.MergeException):
223      results_merger.merge_test_results([
224          extend(GOOD_JSON_TEST_RESULT_0, {'extra': True}),
225          ])
226
227    with self.assertRaises(results_merger.MergeException):
228      results_merger.merge_test_results([
229          GOOD_JSON_TEST_RESULT_0,
230          extend(GOOD_JSON_TEST_RESULT_1, {'extra': True}),
231          ])
232
233  def test_merge_json_test_results_missing_required(self):
234    with self.assertRaises(results_merger.MergeException):
235      results_merger.merge_test_results([
236          remove(GOOD_JSON_TEST_RESULT_0, ['interrupted']),
237          ])
238
239    with self.assertRaises(results_merger.MergeException):
240      results_merger.merge_test_results([
241          GOOD_JSON_TEST_RESULT_0,
242          remove(GOOD_JSON_TEST_RESULT_1, ['interrupted']),
243          ])
244
245  def test_merge_json_test_results_multiple(self):
246    self.assertEqual(
247        results_merger.merge_test_results([
248            GOOD_JSON_TEST_RESULT_0,
249            GOOD_JSON_TEST_RESULT_1,
250            GOOD_JSON_TEST_RESULT_2,
251            ]),
252        GOOD_JSON_TEST_RESULT_MERGED)
253
254  def test_merge_json_test_results_optional_matches(self):
255    self.assertEqual(
256        results_merger.merge_test_results([
257            extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
258            extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
259            extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '.'}),
260            ]),
261        extend(GOOD_JSON_TEST_RESULT_MERGED, {'path_delimiter': '.'}))
262
263  def test_merge_json_test_results_optional_differs(self):
264    with self.assertRaises(results_merger.MergeException):
265      results_merger.merge_test_results([
266          extend(GOOD_JSON_TEST_RESULT_0, {'path_delimiter': '.'}),
267          extend(GOOD_JSON_TEST_RESULT_1, {'path_delimiter': '.'}),
268          extend(GOOD_JSON_TEST_RESULT_2, {'path_delimiter': '/'}),
269          ])
270
271  def test_merge_json_test_results_optional_count(self):
272    self.assertEqual(
273        results_merger.merge_test_results([
274            extend(GOOD_JSON_TEST_RESULT_0, {'fixable': 1}),
275            extend(GOOD_JSON_TEST_RESULT_1, {'fixable': 2}),
276            extend(GOOD_JSON_TEST_RESULT_2, {'fixable': 3}),
277            ]),
278        extend(GOOD_JSON_TEST_RESULT_MERGED, {'fixable': 6}))
279
280  def test_merge_nothing(self):
281    self.assertEqual(
282        results_merger.merge_test_results([]),
283        {})
284
285# TODO(tansell): Make this test fail properly, currently fails with an
286# AttributeError.
287#  def test_merge_test_name_conflict(self):
288#    self.maxDiff = None  # Show full diff if assertion fail
289#    with self.assertRaises(results_merger.MergeException):
290#      results_merger.merge_test_results(
291#        [GOOD_JSON_TEST_RESULT_0, GOOD_JSON_TEST_RESULT_0])
292
293
294
295if __name__ == '__main__':
296  unittest.main()  # pragma: no cover
297