1#!/usr/bin/env python3 2# 3# Copyright 2019, The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Unittests for event_handler.""" 18 19 20from importlib import reload 21import unittest 22from unittest import mock 23 24from atest.test_runners import atest_tf_test_runner as atf_tr 25from atest.test_runners import event_handler as e_h 26from atest.test_runners import test_runner_base 27 28 29class _Event: 30 31 def __init__(self): 32 self.events = [] 33 34 def get_events(self): 35 return self.events 36 37 def add_test_module_started(self, name): 38 self.events.append(( 39 'TEST_MODULE_STARTED', 40 { 41 'moduleContextFileName': 'serial-util1146216{974}2772610436.ser', 42 'moduleName': name, 43 }, 44 )) 45 return self 46 47 def add_test_module_ended(self, data): 48 self.events.append(('TEST_MODULE_ENDED', data)) 49 return self 50 51 def add_test_run_started(self, name, count): 52 self.events.append(( 53 'TEST_RUN_STARTED', 54 { 55 'testCount': count, 56 'runName': name, 57 }, 58 )) 59 return self 60 61 def add_test_run_failed(self, reason): 62 self.events.append(( 63 'TEST_RUN_FAILED', 64 { 65 'reason': reason, 66 }, 67 )) 68 return self 69 70 def add_test_run_ended(self, data): 71 self.events.append(('TEST_RUN_ENDED', data)) 72 return self 73 74 def add_test_started(self, start_time, class_name, test_name): 75 self.events.append(( 76 'TEST_STARTED', 77 { 78 'start_time': start_time, 79 'className': class_name, 80 'testName': test_name, 81 }, 82 )) 83 return self 84 85 def add_test_ignored(self, class_name, test_name, trace): 86 self.events.append(( 87 'TEST_IGNORED', 88 { 89 'className': class_name, 90 'testName': test_name, 91 'trace': trace, 92 }, 93 )) 94 return self 95 96 def add_test_ended(self, end_time, class_name, test_name, **kwargs): 97 self.events.append(( 98 'TEST_ENDED', 99 { 100 'end_time': end_time, 101 'className': class_name, 102 'testName': test_name, 103 } 104 | kwargs, 105 )) 106 return self 107 108 def add_test_failed(self, class_name, test_name, trace): 109 self.events.append(( 110 'TEST_FAILED', 111 { 112 'className': class_name, 113 'testName': test_name, 114 'trace': trace, 115 }, 116 )) 117 return self 118 119 def add_invocation_failed(self, reason): 120 self.events.append(( 121 'INVOCATION_FAILED', 122 { 123 'cause': reason, 124 }, 125 )) 126 return self 127 128 129class EventHandlerUnittests(unittest.TestCase): 130 """Unit tests for event_handler.py""" 131 132 def setUp(self): 133 reload(e_h) 134 self.mock_reporter = mock.Mock() 135 self.fake_eh = e_h.EventHandler( 136 self.mock_reporter, atf_tr.AtestTradefedTestRunner.NAME 137 ) 138 139 def tearDown(self): 140 mock.patch.stopall() 141 142 def test_process_event_normal_results(self): 143 """Test process_event method for normal test results.""" 144 events = ( 145 _Event() 146 .add_test_module_started('someTestModule') 147 .add_test_run_started('com.android.UnitTests', 2) 148 .add_test_started(52, 'someClassName', 'someTestName') 149 .add_test_ended(1048, 'someClassName', 'someTestName') 150 .add_test_started(48, 'someClassName2', 'someTestName2') 151 .add_test_failed('someClassName2', 'someTestName2', 'someTrace') 152 .add_test_ended(9876450, 'someClassName2', 'someTestName2') 153 .add_test_run_ended({}) 154 .add_test_module_ended({'foo': 'bar'}) 155 .get_events() 156 ) 157 for name, data in events: 158 self.fake_eh.process_event(name, data) 159 call1 = mock.call( 160 test_runner_base.TestResult( 161 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 162 group_name='someTestModule', 163 test_name='someClassName#someTestName', 164 status=test_runner_base.PASSED_STATUS, 165 details=None, 166 test_count=1, 167 test_time='(996ms)', 168 runner_total=None, 169 group_total=2, 170 additional_info={}, 171 test_run_name='com.android.UnitTests', 172 ) 173 ) 174 call2 = mock.call( 175 test_runner_base.TestResult( 176 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 177 group_name='someTestModule', 178 test_name='someClassName2#someTestName2', 179 status=test_runner_base.FAILED_STATUS, 180 details='someTrace', 181 test_count=2, 182 test_time='(2h44m36.402s)', 183 runner_total=None, 184 group_total=2, 185 additional_info={}, 186 test_run_name='com.android.UnitTests', 187 ) 188 ) 189 self.mock_reporter.process_test_result.assert_has_calls([call1, call2]) 190 191 def test_process_event_run_failure(self): 192 """Test process_event method run failure.""" 193 events = ( 194 _Event() 195 .add_test_module_started('someTestModule') 196 .add_test_run_started('com.android.UnitTests', 2) 197 .add_test_started(10, 'someClassName', 'someTestName') 198 .add_test_run_failed('someRunFailureReason') 199 .get_events() 200 ) 201 for name, data in events: 202 self.fake_eh.process_event(name, data) 203 call = mock.call( 204 test_runner_base.TestResult( 205 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 206 group_name='someTestModule', 207 test_name='someClassName#someTestName', 208 status=test_runner_base.ERROR_STATUS, 209 details='someRunFailureReason', 210 test_count=1, 211 test_time='', 212 runner_total=None, 213 group_total=2, 214 additional_info={}, 215 test_run_name='com.android.UnitTests', 216 ) 217 ) 218 self.mock_reporter.process_test_result.assert_has_calls([call]) 219 220 def test_process_event_invocation_failure(self): 221 """Test process_event method with invocation failure.""" 222 events = ( 223 _Event() 224 .add_test_run_started('com.android.UnitTests', None) 225 .add_invocation_failed('someInvocationFailureReason') 226 .get_events() 227 ) 228 for name, data in events: 229 self.fake_eh.process_event(name, data) 230 call = mock.call( 231 test_runner_base.TestResult( 232 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 233 group_name=None, 234 test_name=None, 235 status=test_runner_base.ERROR_STATUS, 236 details='someInvocationFailureReason', 237 test_count=0, 238 test_time='', 239 runner_total=None, 240 group_total=None, 241 additional_info={}, 242 test_run_name='com.android.UnitTests', 243 ) 244 ) 245 self.mock_reporter.process_test_result.assert_has_calls([call]) 246 247 def test_process_event_missing_test_run_started_event(self): 248 """Test process_event method for normal test results.""" 249 events = ( 250 _Event() 251 .add_test_started(52, 'someClassName', 'someTestName') 252 .add_test_ended(1048, 'someClassName', 'someTestName') 253 .get_events() 254 ) 255 for name, data in events: 256 self.fake_eh.process_event(name, data) 257 call = mock.call( 258 test_runner_base.TestResult( 259 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 260 group_name=None, 261 test_name='someClassName#someTestName', 262 status=test_runner_base.PASSED_STATUS, 263 details=None, 264 test_count=1, 265 test_time='(996ms)', 266 runner_total=None, 267 group_total=None, 268 additional_info={}, 269 test_run_name=None, 270 ) 271 ) 272 self.mock_reporter.process_test_result.assert_has_calls([call]) 273 274 # pylint: disable=protected-access 275 def test_process_event_test_run_end_without_test_end_throws(self): 276 """Test process_event method with start/end event name not balanced.""" 277 events = ( 278 _Event() 279 .add_test_module_started('someTestModule') 280 .add_test_run_started('com.android.UnitTests', 2) 281 .add_test_started(10, 'someClassName', 'someTestName') 282 .add_test_ended(18, 'someClassName', 'someTestName') 283 .add_test_started(19, 'someClassName', 'someTestName') 284 .add_test_failed('someClassName2', 'someTestName2', 'someTrace') 285 .get_events() 286 ) 287 288 for name, data in events: 289 self.fake_eh.process_event(name, data) 290 call = mock.call( 291 test_runner_base.TestResult( 292 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 293 group_name='someTestModule', 294 test_name='someClassName#someTestName', 295 status=test_runner_base.PASSED_STATUS, 296 details=None, 297 test_count=1, 298 test_time='(8ms)', 299 runner_total=None, 300 group_total=2, 301 additional_info={}, 302 test_run_name='com.android.UnitTests', 303 ) 304 ) 305 self.mock_reporter.process_test_result.assert_has_calls([call]) 306 # Event pair: TEST_STARTED -> TEST_RUN_ENDED 307 # It should raise TradeFedExitError in _check_events_are_balanced() 308 name = 'TEST_RUN_ENDED' 309 data = {} 310 self.assertRaises( 311 e_h.EventHandleError, 312 self.fake_eh._check_events_are_balanced, 313 name, 314 data, 315 self.mock_reporter, 316 ) 317 318 def test_process_event_module_end_without_test_run_end_no_throw(self): 319 """Test process_event method with start/end event name not balanced.""" 320 events = ( 321 _Event() 322 .add_test_module_started('someTestModule') 323 .add_test_run_started('com.android.UnitTests', 2) 324 .add_test_module_ended({'foo': 'bar'}) 325 .get_events() 326 ) 327 for name, data in events[:-1]: 328 self.fake_eh.process_event(name, data) 329 330 self.fake_eh.process_event(*events[-1]) 331 332 def test_process_event_run_end_without_test_end_no_throw(self): 333 """Test process_event method with start/end event name not balanced.""" 334 events = ( 335 _Event() 336 .add_test_module_started('someTestModule') 337 .add_test_run_started('com.android.UnitTests', 2) 338 .add_test_started(10, 'someClassName', 'someTestName') 339 .add_test_run_ended({}) 340 .get_events() 341 ) 342 for name, data in events[:-1]: 343 self.fake_eh.process_event(name, data) 344 345 self.fake_eh.process_event(*events[-1]) 346 347 def test_process_event_ignore(self): 348 """Test _process_event method for normal test results.""" 349 events = ( 350 _Event() 351 .add_test_module_started('someTestModule') 352 .add_test_run_started('com.android.UnitTests', 2) 353 .add_test_started(8, 'someClassName', 'someTestName') 354 .add_test_ended(18, 'someClassName', 'someTestName') 355 .add_test_started(28, 'someClassName2', 'someTestName2') 356 .add_test_ignored('someClassName2', 'someTestName2', 'someTrace') 357 .add_test_ended(90, 'someClassName2', 'someTestName2') 358 .add_test_run_ended({}) 359 .add_test_module_ended({'foo': 'bar'}) 360 .get_events() 361 ) 362 for name, data in events: 363 self.fake_eh.process_event(name, data) 364 call1 = mock.call( 365 test_runner_base.TestResult( 366 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 367 group_name='someTestModule', 368 test_name='someClassName#someTestName', 369 status=test_runner_base.PASSED_STATUS, 370 details=None, 371 test_count=1, 372 test_time='(10ms)', 373 runner_total=None, 374 group_total=2, 375 additional_info={}, 376 test_run_name='com.android.UnitTests', 377 ) 378 ) 379 call2 = mock.call( 380 test_runner_base.TestResult( 381 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 382 group_name='someTestModule', 383 test_name='someClassName2#someTestName2', 384 status=test_runner_base.IGNORED_STATUS, 385 details=None, 386 test_count=2, 387 test_time='(62ms)', 388 runner_total=None, 389 group_total=2, 390 additional_info={}, 391 test_run_name='com.android.UnitTests', 392 ) 393 ) 394 self.mock_reporter.process_test_result.assert_has_calls([call1, call2]) 395 396 def test_process_event_with_additional_info(self): 397 """Test process_event method with perf information.""" 398 events = ( 399 _Event() 400 .add_test_module_started('someTestModule') 401 .add_test_run_started('com.android.UnitTests', 2) 402 .add_test_started(52, 'someClassName', 'someTestName') 403 .add_test_ended(1048, 'someClassName', 'someTestName') 404 .add_test_started(48, 'someClassName2', 'someTestName2') 405 .add_test_failed('someClassName2', 'someTestName2', 'someTrace') 406 .add_test_ended( 407 9876450, 408 'someClassName2', 409 'someTestName2', 410 cpu_time='1234.1234(ns)', 411 real_time='5678.5678(ns)', 412 iterations='6666', 413 ) 414 .add_test_started(10, 'someClassName3', 'someTestName3') 415 .add_test_ended( 416 70, 417 'someClassName3', 418 'someTestName3', 419 additional_info_min='102773', 420 additional_info_mean='105973', 421 additional_info_median='103778', 422 ) 423 .add_test_run_ended({}) 424 .add_test_module_ended({'foo': 'bar'}) 425 .get_events() 426 ) 427 for name, data in events: 428 self.fake_eh.process_event(name, data) 429 call1 = mock.call( 430 test_runner_base.TestResult( 431 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 432 group_name='someTestModule', 433 test_name='someClassName#someTestName', 434 status=test_runner_base.PASSED_STATUS, 435 details=None, 436 test_count=1, 437 test_time='(996ms)', 438 runner_total=None, 439 group_total=2, 440 additional_info={}, 441 test_run_name='com.android.UnitTests', 442 ) 443 ) 444 445 test_additional_info = { 446 'cpu_time': '1234.1234(ns)', 447 'real_time': '5678.5678(ns)', 448 'iterations': '6666', 449 } 450 call2 = mock.call( 451 test_runner_base.TestResult( 452 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 453 group_name='someTestModule', 454 test_name='someClassName2#someTestName2', 455 status=test_runner_base.FAILED_STATUS, 456 details='someTrace', 457 test_count=2, 458 test_time='(2h44m36.402s)', 459 runner_total=None, 460 group_total=2, 461 additional_info=test_additional_info, 462 test_run_name='com.android.UnitTests', 463 ) 464 ) 465 466 test_additional_info2 = { 467 'additional_info_min': '102773', 468 'additional_info_mean': '105973', 469 'additional_info_median': '103778', 470 } 471 call3 = mock.call( 472 test_runner_base.TestResult( 473 runner_name=atf_tr.AtestTradefedTestRunner.NAME, 474 group_name='someTestModule', 475 test_name='someClassName3#someTestName3', 476 status=test_runner_base.PASSED_STATUS, 477 details=None, 478 test_count=3, 479 test_time='(60ms)', 480 runner_total=None, 481 group_total=2, 482 additional_info=test_additional_info2, 483 test_run_name='com.android.UnitTests', 484 ) 485 ) 486 self.mock_reporter.process_test_result.assert_has_calls( 487 [call1, call2, call3] 488 ) 489 490 491if __name__ == '__main__': 492 unittest.main() 493