1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3# Copyright 2020 The ChromiumOS Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Tests for bisecting tool.""" 8 9 10__author__ = "[email protected] (Han Shen)" 11 12import os 13import random 14import sys 15import unittest 16 17from cros_utils import command_executer 18from binary_search_tool import binary_search_state 19from binary_search_tool import run_bisect 20 21from binary_search_tool.test import common 22from binary_search_tool.test import gen_obj 23 24 25def GenObj(): 26 obj_num = random.randint(100, 1000) 27 bad_obj_num = random.randint(obj_num // 100, obj_num // 20) 28 if bad_obj_num == 0: 29 bad_obj_num = 1 30 gen_obj.Main(["--obj_num", str(obj_num), "--bad_obj_num", str(bad_obj_num)]) 31 32 33def CleanObj(): 34 os.remove(common.OBJECTS_FILE) 35 os.remove(common.WORKING_SET_FILE) 36 print( 37 'Deleted "{0}" and "{1}"'.format( 38 common.OBJECTS_FILE, common.WORKING_SET_FILE 39 ) 40 ) 41 42 43class BisectTest(unittest.TestCase): 44 """Tests for run_bisect.py""" 45 46 def setUp(self): 47 with open("./is_setup", "w", encoding="utf-8"): 48 pass 49 50 try: 51 os.remove(binary_search_state.STATE_FILE) 52 except OSError: 53 pass 54 55 def tearDown(self): 56 try: 57 os.remove("./is_setup") 58 os.remove(os.readlink(binary_search_state.STATE_FILE)) 59 os.remove(binary_search_state.STATE_FILE) 60 except OSError: 61 pass 62 63 class FullBisector(run_bisect.Bisector): 64 """Test bisector to test run_bisect.py with""" 65 66 def __init__(self, options, overrides): 67 super(BisectTest.FullBisector, self).__init__(options, overrides) 68 69 def PreRun(self): 70 GenObj() 71 return 0 72 73 def Run(self): 74 return binary_search_state.Run( 75 get_initial_items="./gen_init_list.py", 76 switch_to_good="./switch_to_good.py", 77 switch_to_bad="./switch_to_bad.py", 78 test_script="./is_good.py", 79 prune=True, 80 file_args=True, 81 ) 82 83 def PostRun(self): 84 CleanObj() 85 return 0 86 87 def test_full_bisector(self): 88 ret = run_bisect.Run(self.FullBisector({}, {})) 89 self.assertEqual(ret, 0) 90 self.assertFalse(os.path.exists(common.OBJECTS_FILE)) 91 self.assertFalse(os.path.exists(common.WORKING_SET_FILE)) 92 93 def check_output(self): 94 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 95 ( 96 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 97 "tail -n1" 98 ) 99 ) 100 ls = out.splitlines() 101 self.assertEqual(len(ls), 1) 102 line = ls[0] 103 104 _, _, bad_ones = line.partition("Bad items are: ") 105 bad_ones = bad_ones.split() 106 expected_result = common.ReadObjectsFile() 107 108 # Reconstruct objects file from bad_ones and compare 109 actual_result = [0] * len(expected_result) 110 for bad_obj in bad_ones: 111 actual_result[int(bad_obj)] = 1 112 113 self.assertEqual(actual_result, expected_result) 114 115 116class BisectingUtilsTest(unittest.TestCase): 117 """Tests for bisecting tool.""" 118 119 def setUp(self): 120 """Generate [100-1000] object files, and 1-5% of which are bad ones.""" 121 GenObj() 122 123 with open("./is_setup", "w", encoding="utf-8"): 124 pass 125 126 try: 127 os.remove(binary_search_state.STATE_FILE) 128 except OSError: 129 pass 130 131 def tearDown(self): 132 """Cleanup temp files.""" 133 CleanObj() 134 135 try: 136 os.remove(os.readlink(binary_search_state.STATE_FILE)) 137 except OSError: 138 pass 139 140 cleanup_list = [ 141 "./is_setup", 142 binary_search_state.STATE_FILE, 143 "noinc_prune_bad", 144 "noinc_prune_good", 145 "./cmd_script.sh", 146 ] 147 for f in cleanup_list: 148 if os.path.exists(f): 149 os.remove(f) 150 151 def runTest(self): 152 ret = binary_search_state.Run( 153 get_initial_items="./gen_init_list.py", 154 switch_to_good="./switch_to_good.py", 155 switch_to_bad="./switch_to_bad.py", 156 test_script="./is_good.py", 157 prune=True, 158 file_args=True, 159 ) 160 self.assertEqual(ret, 0) 161 self.check_output() 162 163 def test_arg_parse(self): 164 args = [ 165 "--get_initial_items", 166 "./gen_init_list.py", 167 "--switch_to_good", 168 "./switch_to_good.py", 169 "--switch_to_bad", 170 "./switch_to_bad.py", 171 "--test_script", 172 "./is_good.py", 173 "--prune", 174 "--file_args", 175 ] 176 ret = binary_search_state.Main(args) 177 self.assertEqual(ret, 0) 178 self.check_output() 179 180 def test_test_setup_script(self): 181 os.remove("./is_setup") 182 with self.assertRaises(AssertionError): 183 ret = binary_search_state.Run( 184 get_initial_items="./gen_init_list.py", 185 switch_to_good="./switch_to_good.py", 186 switch_to_bad="./switch_to_bad.py", 187 test_script="./is_good.py", 188 prune=True, 189 file_args=True, 190 ) 191 192 ret = binary_search_state.Run( 193 get_initial_items="./gen_init_list.py", 194 switch_to_good="./switch_to_good.py", 195 switch_to_bad="./switch_to_bad.py", 196 test_script="./is_good.py", 197 test_setup_script="./test_setup.py", 198 prune=True, 199 file_args=True, 200 ) 201 self.assertEqual(ret, 0) 202 self.check_output() 203 204 def test_bad_test_setup_script(self): 205 with self.assertRaises(AssertionError): 206 binary_search_state.Run( 207 get_initial_items="./gen_init_list.py", 208 switch_to_good="./switch_to_good.py", 209 switch_to_bad="./switch_to_bad.py", 210 test_script="./is_good.py", 211 test_setup_script="./test_setup_bad.py", 212 prune=True, 213 file_args=True, 214 ) 215 216 def test_bad_save_state(self): 217 state_file = binary_search_state.STATE_FILE 218 hidden_state_file = os.path.basename( 219 binary_search_state.HIDDEN_STATE_FILE 220 ) 221 222 with open(state_file, "w", encoding="utf-8") as f: 223 f.write("test123") 224 225 bss = binary_search_state.MockBinarySearchState() 226 with self.assertRaises(OSError): 227 bss.SaveState() 228 229 with open(state_file, "r", encoding="utf-8") as f: 230 self.assertEqual(f.read(), "test123") 231 232 os.remove(state_file) 233 234 # Cleanup generated save state that has no symlink 235 files = os.listdir(os.getcwd()) 236 save_states = [x for x in files if x.startswith(hidden_state_file)] 237 _ = [os.remove(x) for x in save_states] 238 239 def test_save_state(self): 240 state_file = binary_search_state.STATE_FILE 241 242 bss = binary_search_state.MockBinarySearchState() 243 bss.SaveState() 244 self.assertTrue(os.path.exists(state_file)) 245 first_state = os.readlink(state_file) 246 247 bss.SaveState() 248 second_state = os.readlink(state_file) 249 self.assertTrue(os.path.exists(state_file)) 250 self.assertTrue(second_state != first_state) 251 self.assertFalse(os.path.exists(first_state)) 252 253 bss.RemoveState() 254 self.assertFalse(os.path.islink(state_file)) 255 self.assertFalse(os.path.exists(second_state)) 256 257 def test_load_state(self): 258 test_items = [1, 2, 3, 4, 5] 259 260 bss = binary_search_state.MockBinarySearchState() 261 bss.all_items = test_items 262 bss.currently_good_items = set([1, 2, 3]) 263 bss.currently_bad_items = set([4, 5]) 264 bss.SaveState() 265 266 bss = None 267 268 bss2 = binary_search_state.MockBinarySearchState.LoadState() 269 self.assertEqual(bss2.all_items, test_items) 270 self.assertEqual(bss2.currently_good_items, set([])) 271 self.assertEqual(bss2.currently_bad_items, set([])) 272 273 def test_tmp_cleanup(self): 274 bss = binary_search_state.MockBinarySearchState( 275 get_initial_items='echo "0\n1\n2\n3"', 276 switch_to_good="./switch_tmp.py", 277 file_args=True, 278 ) 279 bss.SwitchToGood(["0", "1", "2", "3"]) 280 281 tmp_file = None 282 with open("tmp_file", "r", encoding="utf-8") as f: 283 tmp_file = f.read() 284 os.remove("tmp_file") 285 286 self.assertFalse(os.path.exists(tmp_file)) 287 ws = common.ReadWorkingSet() 288 for i in range(3): 289 self.assertEqual(ws[i], 42) 290 291 def test_verify_fail(self): 292 bss = binary_search_state.MockBinarySearchState( 293 get_initial_items="./gen_init_list.py", 294 switch_to_good="./switch_to_bad.py", 295 switch_to_bad="./switch_to_good.py", 296 test_script="./is_good.py", 297 prune=True, 298 file_args=True, 299 verify=True, 300 ) 301 with self.assertRaises(AssertionError): 302 bss.DoVerify() 303 304 def test_early_terminate(self): 305 bss = binary_search_state.MockBinarySearchState( 306 get_initial_items="./gen_init_list.py", 307 switch_to_good="./switch_to_good.py", 308 switch_to_bad="./switch_to_bad.py", 309 test_script="./is_good.py", 310 prune=True, 311 file_args=True, 312 iterations=1, 313 ) 314 bss.DoSearchBadItems() 315 self.assertFalse(bss.found_items) 316 317 def test_no_prune(self): 318 bss = binary_search_state.MockBinarySearchState( 319 get_initial_items="./gen_init_list.py", 320 switch_to_good="./switch_to_good.py", 321 switch_to_bad="./switch_to_bad.py", 322 test_script="./is_good.py", 323 test_setup_script="./test_setup.py", 324 prune=False, 325 file_args=True, 326 ) 327 bss.DoSearchBadItems() 328 self.assertEqual(len(bss.found_items), 1) 329 330 bad_objs = common.ReadObjectsFile() 331 found_obj = int(bss.found_items.pop()) 332 self.assertEqual(bad_objs[found_obj], 1) 333 334 def test_set_file(self): 335 binary_search_state.Run( 336 get_initial_items="./gen_init_list.py", 337 switch_to_good="./switch_to_good_set_file.py", 338 switch_to_bad="./switch_to_bad_set_file.py", 339 test_script="./is_good.py", 340 prune=True, 341 file_args=True, 342 verify=True, 343 ) 344 self.check_output() 345 346 def test_noincremental_prune(self): 347 ret = binary_search_state.Run( 348 get_initial_items="./gen_init_list.py", 349 switch_to_good="./switch_to_good_noinc_prune.py", 350 switch_to_bad="./switch_to_bad_noinc_prune.py", 351 test_script="./is_good_noinc_prune.py", 352 test_setup_script="./test_setup.py", 353 prune=True, 354 noincremental=True, 355 file_args=True, 356 verify=False, 357 ) 358 self.assertEqual(ret, 0) 359 self.check_output() 360 361 def check_output(self): 362 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 363 ( 364 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 365 "tail -n1" 366 ) 367 ) 368 ls = out.splitlines() 369 self.assertEqual(len(ls), 1) 370 line = ls[0] 371 372 _, _, bad_ones = line.partition("Bad items are: ") 373 bad_ones = bad_ones.split() 374 expected_result = common.ReadObjectsFile() 375 376 # Reconstruct objects file from bad_ones and compare 377 actual_result = [0] * len(expected_result) 378 for bad_obj in bad_ones: 379 actual_result[int(bad_obj)] = 1 380 381 self.assertEqual(actual_result, expected_result) 382 383 384class BisectingUtilsPassTest(BisectingUtilsTest): 385 """Tests for bisecting tool at pass/transformation level.""" 386 387 def check_pass_output(self, pass_name, pass_num, trans_num): 388 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 389 ( 390 'grep "Bad pass: " logs/binary_search_tool_test.py.out | ' 391 "tail -n1" 392 ) 393 ) 394 ls = out.splitlines() 395 self.assertEqual(len(ls), 1) 396 line = ls[0] 397 _, _, bad_info = line.partition("Bad pass: ") 398 actual_info = pass_name + " at number " + str(pass_num) 399 self.assertEqual(actual_info, bad_info) 400 401 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 402 ( 403 'grep "Bad transformation number: ' 404 '" logs/binary_search_tool_test.py.out | ' 405 "tail -n1" 406 ) 407 ) 408 ls = out.splitlines() 409 self.assertEqual(len(ls), 1) 410 line = ls[0] 411 _, _, bad_info = line.partition("Bad transformation number: ") 412 actual_info = str(trans_num) 413 self.assertEqual(actual_info, bad_info) 414 415 def test_with_prune(self): 416 ret = binary_search_state.Run( 417 get_initial_items="./gen_init_list.py", 418 switch_to_good="./switch_to_good.py", 419 switch_to_bad="./switch_to_bad.py", 420 test_script="./is_good.py", 421 pass_bisect="./generate_cmd.py", 422 prune=True, 423 file_args=True, 424 ) 425 self.assertEqual(ret, 1) 426 427 def test_gen_cmd_script(self): 428 bss = binary_search_state.MockBinarySearchState( 429 get_initial_items="./gen_init_list.py", 430 switch_to_good="./switch_to_good.py", 431 switch_to_bad="./switch_to_bad.py", 432 test_script="./is_good.py", 433 pass_bisect="./generate_cmd.py", 434 prune=False, 435 file_args=True, 436 ) 437 bss.DoSearchBadItems() 438 cmd_script_path = bss.cmd_script 439 self.assertTrue(os.path.exists(cmd_script_path)) 440 441 def test_no_pass_support(self): 442 bss = binary_search_state.MockBinarySearchState( 443 get_initial_items="./gen_init_list.py", 444 switch_to_good="./switch_to_good.py", 445 switch_to_bad="./switch_to_bad.py", 446 test_script="./is_good.py", 447 pass_bisect="./generate_cmd.py", 448 prune=False, 449 file_args=True, 450 ) 451 bss.cmd_script = "./cmd_script_no_support.py" 452 # No support for -opt-bisect-limit 453 with self.assertRaises(RuntimeError): 454 bss.BuildWithPassLimit(-1) 455 456 def test_no_transform_support(self): 457 bss = binary_search_state.MockBinarySearchState( 458 get_initial_items="./gen_init_list.py", 459 switch_to_good="./switch_to_good.py", 460 switch_to_bad="./switch_to_bad.py", 461 test_script="./is_good.py", 462 pass_bisect="./generate_cmd.py", 463 prune=False, 464 file_args=True, 465 ) 466 bss.cmd_script = "./cmd_script_no_support.py" 467 # No support for -print-debug-counter 468 with self.assertRaises(RuntimeError): 469 bss.BuildWithTransformLimit(-1, "counter_name") 470 471 def test_pass_transform_bisect(self): 472 bss = binary_search_state.MockBinarySearchState( 473 get_initial_items="./gen_init_list.py", 474 switch_to_good="./switch_to_good.py", 475 switch_to_bad="./switch_to_bad.py", 476 test_script="./is_good.py", 477 pass_bisect="./generate_cmd.py", 478 prune=False, 479 file_args=True, 480 ) 481 pass_num = 4 482 trans_num = 19 483 bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) 484 bss.DoSearchBadPass() 485 self.check_pass_output("instcombine-visit", pass_num, trans_num) 486 487 def test_result_not_reproduced_pass(self): 488 bss = binary_search_state.MockBinarySearchState( 489 get_initial_items="./gen_init_list.py", 490 switch_to_good="./switch_to_good.py", 491 switch_to_bad="./switch_to_bad.py", 492 test_script="./is_good.py", 493 pass_bisect="./generate_cmd.py", 494 prune=False, 495 file_args=True, 496 ) 497 # Fails reproducing at pass level. 498 pass_num = 0 499 trans_num = 19 500 bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) 501 with self.assertRaises(ValueError): 502 bss.DoSearchBadPass() 503 504 def test_result_not_reproduced_transform(self): 505 bss = binary_search_state.MockBinarySearchState( 506 get_initial_items="./gen_init_list.py", 507 switch_to_good="./switch_to_good.py", 508 switch_to_bad="./switch_to_bad.py", 509 test_script="./is_good.py", 510 pass_bisect="./generate_cmd.py", 511 prune=False, 512 file_args=True, 513 ) 514 # Fails reproducing at transformation level. 515 pass_num = 4 516 trans_num = 0 517 bss.cmd_script = "./cmd_script.py %d %d" % (pass_num, trans_num) 518 with self.assertRaises(ValueError): 519 bss.DoSearchBadPass() 520 521 522class BisectStressTest(unittest.TestCase): 523 """Stress tests for bisecting tool.""" 524 525 def test_every_obj_bad(self): 526 amt = 25 527 gen_obj.Main(["--obj_num", str(amt), "--bad_obj_num", str(amt)]) 528 ret = binary_search_state.Run( 529 get_initial_items="./gen_init_list.py", 530 switch_to_good="./switch_to_good.py", 531 switch_to_bad="./switch_to_bad.py", 532 test_script="./is_good.py", 533 prune=True, 534 file_args=True, 535 verify=False, 536 ) 537 self.assertEqual(ret, 0) 538 self.check_output() 539 540 def test_every_index_is_bad(self): 541 amt = 25 542 for i in range(amt): 543 obj_list = ["0"] * amt 544 obj_list[i] = "1" 545 obj_list = ",".join(obj_list) 546 gen_obj.Main(["--obj_list", obj_list]) 547 ret = binary_search_state.Run( 548 get_initial_items="./gen_init_list.py", 549 switch_to_good="./switch_to_good.py", 550 switch_to_bad="./switch_to_bad.py", 551 test_setup_script="./test_setup.py", 552 test_script="./is_good.py", 553 prune=True, 554 file_args=True, 555 ) 556 self.assertEqual(ret, 0) 557 self.check_output() 558 559 def check_output(self): 560 _, out, _ = command_executer.GetCommandExecuter().RunCommandWOutput( 561 ( 562 'grep "Bad items are: " logs/binary_search_tool_test.py.out | ' 563 "tail -n1" 564 ) 565 ) 566 ls = out.splitlines() 567 self.assertEqual(len(ls), 1) 568 line = ls[0] 569 570 _, _, bad_ones = line.partition("Bad items are: ") 571 bad_ones = bad_ones.split() 572 expected_result = common.ReadObjectsFile() 573 574 # Reconstruct objects file from bad_ones and compare 575 actual_result = [0] * len(expected_result) 576 for bad_obj in bad_ones: 577 actual_result[int(bad_obj)] = 1 578 579 self.assertEqual(actual_result, expected_result) 580 581 582def Main(argv): 583 num_tests = 2 584 if len(argv) > 1: 585 num_tests = int(argv[1]) 586 587 suite = unittest.TestSuite() 588 for _ in range(0, num_tests): 589 suite.addTest(BisectingUtilsTest()) 590 suite.addTest(BisectingUtilsTest("test_arg_parse")) 591 suite.addTest(BisectingUtilsTest("test_test_setup_script")) 592 suite.addTest(BisectingUtilsTest("test_bad_test_setup_script")) 593 suite.addTest(BisectingUtilsTest("test_bad_save_state")) 594 suite.addTest(BisectingUtilsTest("test_save_state")) 595 suite.addTest(BisectingUtilsTest("test_load_state")) 596 suite.addTest(BisectingUtilsTest("test_tmp_cleanup")) 597 suite.addTest(BisectingUtilsTest("test_verify_fail")) 598 suite.addTest(BisectingUtilsTest("test_early_terminate")) 599 suite.addTest(BisectingUtilsTest("test_no_prune")) 600 suite.addTest(BisectingUtilsTest("test_set_file")) 601 suite.addTest(BisectingUtilsTest("test_noincremental_prune")) 602 suite.addTest(BisectingUtilsPassTest("test_with_prune")) 603 suite.addTest(BisectingUtilsPassTest("test_gen_cmd_script")) 604 suite.addTest(BisectingUtilsPassTest("test_no_pass_support")) 605 suite.addTest(BisectingUtilsPassTest("test_no_transform_support")) 606 suite.addTest(BisectingUtilsPassTest("test_pass_transform_bisect")) 607 suite.addTest(BisectingUtilsPassTest("test_result_not_reproduced_pass")) 608 suite.addTest( 609 BisectingUtilsPassTest("test_result_not_reproduced_transform") 610 ) 611 suite.addTest(BisectTest("test_full_bisector")) 612 suite.addTest(BisectStressTest("test_every_obj_bad")) 613 suite.addTest(BisectStressTest("test_every_index_is_bad")) 614 runner = unittest.TextTestRunner() 615 runner.run(suite) 616 617 618if __name__ == "__main__": 619 Main(sys.argv) 620