xref: /aosp_15_r20/external/cronet/build/android/pylib/results/json_results_test.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b) 
1#!/usr/bin/env vpython3
2# Copyright 2014 The Chromium Authors
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6
7import unittest
8
9from pylib.base import base_test_result
10from pylib.results import json_results
11
12
13class JsonResultsTest(unittest.TestCase):
14
15  def testGenerateResultsDict_passedResult(self):
16    result = base_test_result.BaseTestResult(
17        'test.package.TestName', base_test_result.ResultType.PASS)
18
19    all_results = base_test_result.TestRunResults()
20    all_results.AddResult(result)
21
22    results_dict = json_results.GenerateResultsDict([all_results])
23    self.assertEqual(['test.package.TestName'], results_dict['all_tests'])
24    self.assertEqual(1, len(results_dict['per_iteration_data']))
25
26    iteration_result = results_dict['per_iteration_data'][0]
27    self.assertTrue('test.package.TestName' in iteration_result)
28    self.assertEqual(1, len(iteration_result['test.package.TestName']))
29
30    test_iteration_result = iteration_result['test.package.TestName'][0]
31    self.assertTrue('status' in test_iteration_result)
32    self.assertEqual('SUCCESS', test_iteration_result['status'])
33
34  def testGenerateResultsDict_skippedResult(self):
35    result = base_test_result.BaseTestResult(
36        'test.package.TestName', base_test_result.ResultType.SKIP)
37
38    all_results = base_test_result.TestRunResults()
39    all_results.AddResult(result)
40
41    results_dict = json_results.GenerateResultsDict([all_results])
42    self.assertEqual(['test.package.TestName'], results_dict['all_tests'])
43    self.assertEqual(1, len(results_dict['per_iteration_data']))
44
45    iteration_result = results_dict['per_iteration_data'][0]
46    self.assertTrue('test.package.TestName' in iteration_result)
47    self.assertEqual(1, len(iteration_result['test.package.TestName']))
48
49    test_iteration_result = iteration_result['test.package.TestName'][0]
50    self.assertTrue('status' in test_iteration_result)
51    self.assertEqual('SKIPPED', test_iteration_result['status'])
52
53  def testGenerateResultsDict_failedResult(self):
54    result = base_test_result.BaseTestResult(
55        'test.package.TestName', base_test_result.ResultType.FAIL)
56
57    all_results = base_test_result.TestRunResults()
58    all_results.AddResult(result)
59
60    results_dict = json_results.GenerateResultsDict([all_results])
61    self.assertEqual(['test.package.TestName'], results_dict['all_tests'])
62    self.assertEqual(1, len(results_dict['per_iteration_data']))
63
64    iteration_result = results_dict['per_iteration_data'][0]
65    self.assertTrue('test.package.TestName' in iteration_result)
66    self.assertEqual(1, len(iteration_result['test.package.TestName']))
67
68    test_iteration_result = iteration_result['test.package.TestName'][0]
69    self.assertTrue('status' in test_iteration_result)
70    self.assertEqual('FAILURE', test_iteration_result['status'])
71
72  def testGenerateResultsDict_duration(self):
73    result = base_test_result.BaseTestResult(
74        'test.package.TestName', base_test_result.ResultType.PASS, duration=123)
75
76    all_results = base_test_result.TestRunResults()
77    all_results.AddResult(result)
78
79    results_dict = json_results.GenerateResultsDict([all_results])
80    self.assertEqual(['test.package.TestName'], results_dict['all_tests'])
81    self.assertEqual(1, len(results_dict['per_iteration_data']))
82
83    iteration_result = results_dict['per_iteration_data'][0]
84    self.assertTrue('test.package.TestName' in iteration_result)
85    self.assertEqual(1, len(iteration_result['test.package.TestName']))
86
87    test_iteration_result = iteration_result['test.package.TestName'][0]
88    self.assertTrue('elapsed_time_ms' in test_iteration_result)
89    self.assertEqual(123, test_iteration_result['elapsed_time_ms'])
90
91  def testGenerateResultsDict_multipleResults(self):
92    result1 = base_test_result.BaseTestResult(
93        'test.package.TestName1', base_test_result.ResultType.PASS)
94    result2 = base_test_result.BaseTestResult(
95        'test.package.TestName2', base_test_result.ResultType.PASS)
96
97    all_results = base_test_result.TestRunResults()
98    all_results.AddResult(result1)
99    all_results.AddResult(result2)
100
101    results_dict = json_results.GenerateResultsDict([all_results])
102    self.assertEqual(['test.package.TestName1', 'test.package.TestName2'],
103                     results_dict['all_tests'])
104
105    self.assertTrue('per_iteration_data' in results_dict)
106    iterations = results_dict['per_iteration_data']
107    self.assertEqual(1, len(iterations))
108
109    expected_tests = set([
110        'test.package.TestName1',
111        'test.package.TestName2',
112    ])
113
114    for test_name, iteration_result in iterations[0].items():
115      self.assertTrue(test_name in expected_tests)
116      expected_tests.remove(test_name)
117      self.assertEqual(1, len(iteration_result))
118
119      test_iteration_result = iteration_result[0]
120      self.assertTrue('status' in test_iteration_result)
121      self.assertEqual('SUCCESS', test_iteration_result['status'])
122
123  def testGenerateResultsDict_passOnRetry(self):
124    raw_results = []
125
126    result1 = base_test_result.BaseTestResult(
127        'test.package.TestName1', base_test_result.ResultType.FAIL)
128    run_results1 = base_test_result.TestRunResults()
129    run_results1.AddResult(result1)
130    raw_results.append(run_results1)
131
132    result2 = base_test_result.BaseTestResult(
133        'test.package.TestName1', base_test_result.ResultType.PASS)
134    run_results2 = base_test_result.TestRunResults()
135    run_results2.AddResult(result2)
136    raw_results.append(run_results2)
137
138    results_dict = json_results.GenerateResultsDict([raw_results])
139    self.assertEqual(['test.package.TestName1'], results_dict['all_tests'])
140
141    # Check that there's only one iteration.
142    self.assertIn('per_iteration_data', results_dict)
143    iterations = results_dict['per_iteration_data']
144    self.assertEqual(1, len(iterations))
145
146    # Check that test.package.TestName1 is the only test in the iteration.
147    self.assertEqual(1, len(iterations[0]))
148    self.assertIn('test.package.TestName1', iterations[0])
149
150    # Check that there are two results for test.package.TestName1.
151    actual_test_results = iterations[0]['test.package.TestName1']
152    self.assertEqual(2, len(actual_test_results))
153
154    # Check that the first result is a failure.
155    self.assertIn('status', actual_test_results[0])
156    self.assertEqual('FAILURE', actual_test_results[0]['status'])
157
158    # Check that the second result is a success.
159    self.assertIn('status', actual_test_results[1])
160    self.assertEqual('SUCCESS', actual_test_results[1]['status'])
161
162  def testGenerateResultsDict_globalTags(self):
163    raw_results = []
164    global_tags = ['UNRELIABLE_RESULTS']
165
166    results_dict = json_results.GenerateResultsDict(
167        [raw_results], global_tags=global_tags)
168    self.assertEqual(['UNRELIABLE_RESULTS'], results_dict['global_tags'])
169
170  def testGenerateResultsDict_loslessSnippet(self):
171    result = base_test_result.BaseTestResult(
172        'test.package.TestName', base_test_result.ResultType.FAIL)
173    log = 'blah-blah'
174    result.SetLog(log)
175
176    all_results = base_test_result.TestRunResults()
177    all_results.AddResult(result)
178
179    results_dict = json_results.GenerateResultsDict([all_results])
180    self.assertEqual(['test.package.TestName'], results_dict['all_tests'])
181    self.assertEqual(1, len(results_dict['per_iteration_data']))
182
183    iteration_result = results_dict['per_iteration_data'][0]
184    self.assertTrue('test.package.TestName' in iteration_result)
185    self.assertEqual(1, len(iteration_result['test.package.TestName']))
186
187    test_iteration_result = iteration_result['test.package.TestName'][0]
188    self.assertTrue('losless_snippet' in test_iteration_result)
189    self.assertTrue(test_iteration_result['losless_snippet'])
190    self.assertTrue('output_snippet' in test_iteration_result)
191    self.assertEqual(log, test_iteration_result['output_snippet'])
192    self.assertTrue('output_snippet_base64' in test_iteration_result)
193    self.assertEqual('', test_iteration_result['output_snippet_base64'])
194
195  def testGenerateJsonTestResultFormatDict_passedResult(self):
196    result = base_test_result.BaseTestResult('test.package.TestName',
197                                             base_test_result.ResultType.PASS)
198
199    all_results = base_test_result.TestRunResults()
200    all_results.AddResult(result)
201
202    results_dict = json_results.GenerateJsonTestResultFormatDict([all_results],
203                                                                 False)
204    self.assertEqual(1, len(results_dict['tests']))
205    self.assertEqual(1, len(results_dict['tests']['test']))
206    self.assertEqual(1, len(results_dict['tests']['test']['package']))
207    self.assertEqual(
208        'PASS',
209        results_dict['tests']['test']['package']['TestName']['expected'])
210    self.assertEqual(
211        'PASS', results_dict['tests']['test']['package']['TestName']['actual'])
212
213    self.assertTrue('FAIL' not in results_dict['num_failures_by_type']
214                    or results_dict['num_failures_by_type']['FAIL'] == 0)
215    self.assertIn('PASS', results_dict['num_failures_by_type'])
216    self.assertEqual(1, results_dict['num_failures_by_type']['PASS'])
217
218  def testGenerateJsonTestResultFormatDict_failedResult(self):
219    result = base_test_result.BaseTestResult('test.package.TestName',
220                                             base_test_result.ResultType.FAIL)
221
222    all_results = base_test_result.TestRunResults()
223    all_results.AddResult(result)
224
225    results_dict = json_results.GenerateJsonTestResultFormatDict([all_results],
226                                                                 False)
227    self.assertEqual(1, len(results_dict['tests']))
228    self.assertEqual(1, len(results_dict['tests']['test']))
229    self.assertEqual(1, len(results_dict['tests']['test']['package']))
230    self.assertEqual(
231        'PASS',
232        results_dict['tests']['test']['package']['TestName']['expected'])
233    self.assertEqual(
234        'FAIL', results_dict['tests']['test']['package']['TestName']['actual'])
235    self.assertEqual(
236        True,
237        results_dict['tests']['test']['package']['TestName']['is_unexpected'])
238
239    self.assertTrue('PASS' not in results_dict['num_failures_by_type']
240                    or results_dict['num_failures_by_type']['PASS'] == 0)
241    self.assertIn('FAIL', results_dict['num_failures_by_type'])
242    self.assertEqual(1, results_dict['num_failures_by_type']['FAIL'])
243
244  def testGenerateJsonTestResultFormatDict_skippedResult(self):
245    result = base_test_result.BaseTestResult('test.package.TestName',
246                                             base_test_result.ResultType.SKIP)
247
248    all_results = base_test_result.TestRunResults()
249    all_results.AddResult(result)
250
251    results_dict = json_results.GenerateJsonTestResultFormatDict([all_results],
252                                                                 False)
253    self.assertEqual(1, len(results_dict['tests']))
254    self.assertEqual(1, len(results_dict['tests']['test']))
255    self.assertEqual(1, len(results_dict['tests']['test']['package']))
256    self.assertEqual(
257        'PASS',
258        results_dict['tests']['test']['package']['TestName']['expected'])
259    self.assertEqual(
260        'SKIP', results_dict['tests']['test']['package']['TestName']['actual'])
261    # Should only be set if the test fails.
262    self.assertNotIn('is_unexpected',
263                     results_dict['tests']['test']['package']['TestName'])
264
265    self.assertTrue('FAIL' not in results_dict['num_failures_by_type']
266                    or results_dict['num_failures_by_type']['FAIL'] == 0)
267    self.assertTrue('PASS' not in results_dict['num_failures_by_type']
268                    or results_dict['num_failures_by_type']['PASS'] == 0)
269    self.assertIn('SKIP', results_dict['num_failures_by_type'])
270    self.assertEqual(1, results_dict['num_failures_by_type']['SKIP'])
271
272  def testGenerateJsonTestResultFormatDict_failedResultWithRetry(self):
273    result_1 = base_test_result.BaseTestResult('test.package.TestName',
274                                               base_test_result.ResultType.FAIL)
275    run_results_1 = base_test_result.TestRunResults()
276    run_results_1.AddResult(result_1)
277
278    # Simulate a second retry with failure.
279    result_2 = base_test_result.BaseTestResult('test.package.TestName',
280                                               base_test_result.ResultType.FAIL)
281    run_results_2 = base_test_result.TestRunResults()
282    run_results_2.AddResult(result_2)
283
284    all_results = [run_results_1, run_results_2]
285
286    results_dict = json_results.GenerateJsonTestResultFormatDict(
287        all_results, False)
288    self.assertEqual(1, len(results_dict['tests']))
289    self.assertEqual(1, len(results_dict['tests']['test']))
290    self.assertEqual(1, len(results_dict['tests']['test']['package']))
291    self.assertEqual(
292        'PASS',
293        results_dict['tests']['test']['package']['TestName']['expected'])
294    self.assertEqual(
295        'FAIL FAIL',
296        results_dict['tests']['test']['package']['TestName']['actual'])
297    self.assertEqual(
298        True,
299        results_dict['tests']['test']['package']['TestName']['is_unexpected'])
300
301    self.assertTrue('PASS' not in results_dict['num_failures_by_type']
302                    or results_dict['num_failures_by_type']['PASS'] == 0)
303    # According to the spec: If a test was run more than once, only the first
304    # invocation's result is included in the totals.
305    self.assertIn('FAIL', results_dict['num_failures_by_type'])
306    self.assertEqual(1, results_dict['num_failures_by_type']['FAIL'])
307
308
309if __name__ == '__main__':
310  unittest.main(verbosity=2)
311