1#!/usr/bin/env python3 2# Copyright 2019 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Tests when updating a tryjob's status.""" 7 8import contextlib 9import json 10import os 11import subprocess 12import unittest 13from unittest import mock 14 15import test_helpers 16import update_tryjob_status 17 18 19class UpdateTryjobStatusTest(unittest.TestCase): 20 """Unittests for updating a tryjob's 'status'.""" 21 22 def testFoundTryjobIndex(self): 23 test_tryjobs = [ 24 { 25 "rev": 123, 26 "url": "https://some_url_to_CL.com", 27 "cl": "https://some_link_to_tryjob.com", 28 "status": "good", 29 "buildbucket_id": 91835, 30 }, 31 { 32 "rev": 1000, 33 "url": "https://some_url_to_CL.com", 34 "cl": "https://some_link_to_tryjob.com", 35 "status": "pending", 36 "buildbucket_id": 10931, 37 }, 38 ] 39 40 expected_index = 0 41 42 revision_to_find = 123 43 44 self.assertEqual( 45 update_tryjob_status.FindTryjobIndex( 46 revision_to_find, test_tryjobs 47 ), 48 expected_index, 49 ) 50 51 def testNotFindTryjobIndex(self): 52 test_tryjobs = [ 53 { 54 "rev": 500, 55 "url": "https://some_url_to_CL.com", 56 "cl": "https://some_link_to_tryjob.com", 57 "status": "bad", 58 "buildbucket_id": 390, 59 }, 60 { 61 "rev": 10, 62 "url": "https://some_url_to_CL.com", 63 "cl": "https://some_link_to_tryjob.com", 64 "status": "skip", 65 "buildbucket_id": 10, 66 }, 67 ] 68 69 revision_to_find = 250 70 71 self.assertIsNone( 72 update_tryjob_status.FindTryjobIndex(revision_to_find, test_tryjobs) 73 ) 74 75 @mock.patch.object(subprocess, "Popen") 76 # Simulate the behavior of `os.rename()` when successfully renamed a file. 77 @mock.patch.object(os, "rename", return_value=None) 78 # Simulate the behavior of `os.path.basename()` when successfully retrieved 79 # the basename of the temp .JSON file. 80 @mock.patch.object(os.path, "basename", return_value="tmpFile.json") 81 def testInvalidExitCodeByCustomScript( 82 self, mock_basename, mock_rename_file, mock_exec_custom_script 83 ): 84 error_message_by_custom_script = "Failed to parse .JSON file" 85 86 # Simulate the behavior of 'subprocess.Popen()' when executing the 87 # custom script. 88 # 89 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 90 popen_result = mock.MagicMock() 91 popen_result.communicate.return_value = ( 92 None, 93 error_message_by_custom_script, 94 ) 95 custom_script_exit_code = 1 96 popen_result.returncode = custom_script_exit_code 97 mock_exec_custom_script.return_value = contextlib.nullcontext( 98 popen_result 99 ) 100 101 tryjob_contents = { 102 "status": "good", 103 "rev": 1234, 104 "url": "https://some_url_to_CL.com", 105 "link": "https://some_url_to_tryjob.com", 106 } 107 108 custom_script_path = "/abs/path/to/script.py" 109 status_file_path = "/abs/path/to/status_file.json" 110 111 name_json_file = os.path.join( 112 os.path.dirname(status_file_path), "tmpFile.json" 113 ) 114 115 expected_error_message = ( 116 "Custom script %s exit code %d did not match " 117 'any of the expected exit codes: %s for "good", ' 118 '%d for "bad", or %d for "skip".\nPlease check ' 119 "%s for information about the tryjob: %s" 120 % ( 121 custom_script_path, 122 custom_script_exit_code, 123 update_tryjob_status.CustomScriptStatus.GOOD.value, 124 update_tryjob_status.CustomScriptStatus.BAD.value, 125 update_tryjob_status.CustomScriptStatus.SKIP.value, 126 name_json_file, 127 error_message_by_custom_script, 128 ) 129 ) 130 131 # Verify the exception is raised when the exit code by the custom script 132 # does not match any of the exit codes in the mapping of 133 # `custom_script_exit_value_mapping`. 134 with self.assertRaises(ValueError) as err: 135 update_tryjob_status.GetCustomScriptResult( 136 custom_script_path, status_file_path, tryjob_contents 137 ) 138 139 self.assertEqual(str(err.exception), expected_error_message) 140 141 mock_exec_custom_script.assert_called_once() 142 143 mock_rename_file.assert_called_once() 144 145 mock_basename.assert_called_once() 146 147 @mock.patch.object(subprocess, "Popen") 148 # Simulate the behavior of `os.rename()` when successfully renamed a file. 149 @mock.patch.object(os, "rename", return_value=None) 150 # Simulate the behavior of `os.path.basename()` when successfully retrieved 151 # the basename of the temp .JSON file. 152 @mock.patch.object(os.path, "basename", return_value="tmpFile.json") 153 def testValidExitCodeByCustomScript( 154 self, mock_basename, mock_rename_file, mock_exec_custom_script 155 ): 156 # Simulate the behavior of 'subprocess.Popen()' when executing the 157 # custom script. 158 # 159 # `Popen.communicate()` returns a tuple of `stdout` and `stderr`. 160 popen_result = mock.MagicMock() 161 popen_result.communicate.return_value = ( 162 None, 163 None, 164 ) 165 popen_result.returncode = ( 166 update_tryjob_status.CustomScriptStatus.GOOD.value 167 ) 168 mock_exec_custom_script.return_value = contextlib.nullcontext( 169 popen_result 170 ) 171 172 tryjob_contents = { 173 "status": "good", 174 "rev": 1234, 175 "url": "https://some_url_to_CL.com", 176 "link": "https://some_url_to_tryjob.com", 177 } 178 179 custom_script_path = "/abs/path/to/script.py" 180 status_file_path = "/abs/path/to/status_file.json" 181 182 self.assertEqual( 183 update_tryjob_status.GetCustomScriptResult( 184 custom_script_path, status_file_path, tryjob_contents 185 ), 186 update_tryjob_status.TryjobStatus.GOOD.value, 187 ) 188 189 mock_exec_custom_script.assert_called_once() 190 191 mock_rename_file.assert_not_called() 192 193 mock_basename.assert_not_called() 194 195 def testNoTryjobsInStatusFileWhenUpdatingTryjobStatus(self): 196 bisect_test_contents = {"start": 369410, "end": 369420, "jobs": []} 197 198 # Create a temporary .JSON file to simulate a .JSON file that has 199 # bisection contents. 200 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 201 with open(temp_json_file, "w", encoding="utf-8") as f: 202 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 203 204 revision_to_update = 369412 205 206 custom_script = None 207 208 # Verify the exception is raised when the `status_file` does not 209 # have any `jobs` (empty). 210 with self.assertRaises(SystemExit) as err: 211 update_tryjob_status.UpdateTryjobStatus( 212 revision_to_update, 213 update_tryjob_status.TryjobStatus.GOOD, 214 temp_json_file, 215 custom_script, 216 ) 217 218 self.assertEqual( 219 str(err.exception), "No tryjobs in %s" % temp_json_file 220 ) 221 222 # Simulate the behavior of `FindTryjobIndex()` when the tryjob does not 223 # exist in the status file. 224 @mock.patch.object( 225 update_tryjob_status, "FindTryjobIndex", return_value=None 226 ) 227 def testNotFindTryjobIndexWhenUpdatingTryjobStatus( 228 self, mock_find_tryjob_index 229 ): 230 bisect_test_contents = { 231 "start": 369410, 232 "end": 369420, 233 "jobs": [{"rev": 369411, "status": "pending"}], 234 } 235 236 # Create a temporary .JSON file to simulate a .JSON file that has 237 # bisection contents. 238 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 239 with open(temp_json_file, "w", encoding="utf-8") as f: 240 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 241 242 revision_to_update = 369416 243 244 custom_script = None 245 246 # Verify the exception is raised when the `status_file` does not 247 # have any `jobs` (empty). 248 with self.assertRaises(ValueError) as err: 249 update_tryjob_status.UpdateTryjobStatus( 250 revision_to_update, 251 update_tryjob_status.TryjobStatus.SKIP, 252 temp_json_file, 253 custom_script, 254 ) 255 256 self.assertEqual( 257 str(err.exception), 258 "Unable to find tryjob for %d in %s" 259 % (revision_to_update, temp_json_file), 260 ) 261 262 mock_find_tryjob_index.assert_called_once() 263 264 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 265 # status file. 266 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 267 def testSuccessfullyUpdatedTryjobStatusToGood(self, mock_find_tryjob_index): 268 bisect_test_contents = { 269 "start": 369410, 270 "end": 369420, 271 "jobs": [{"rev": 369411, "status": "pending"}], 272 } 273 274 # Create a temporary .JSON file to simulate a .JSON file that has 275 # bisection contents. 276 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 277 with open(temp_json_file, "w", encoding="utf-8") as f: 278 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 279 280 revision_to_update = 369411 281 282 # Index of the tryjob that is going to have its 'status' value 283 # updated. 284 tryjob_index = 0 285 286 custom_script = None 287 288 update_tryjob_status.UpdateTryjobStatus( 289 revision_to_update, 290 update_tryjob_status.TryjobStatus.GOOD, 291 temp_json_file, 292 custom_script, 293 ) 294 295 # Verify that the tryjob's 'status' has been updated in the status 296 # file. 297 with open(temp_json_file, encoding="utf-8") as status_file: 298 bisect_contents = json.load(status_file) 299 300 self.assertEqual( 301 bisect_contents["jobs"][tryjob_index]["status"], 302 update_tryjob_status.TryjobStatus.GOOD.value, 303 ) 304 305 mock_find_tryjob_index.assert_called_once() 306 307 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 308 # status file. 309 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 310 def testSuccessfullyUpdatedTryjobStatusToBad(self, mock_find_tryjob_index): 311 bisect_test_contents = { 312 "start": 369410, 313 "end": 369420, 314 "jobs": [{"rev": 369411, "status": "pending"}], 315 } 316 317 # Create a temporary .JSON file to simulate a .JSON file that has 318 # bisection contents. 319 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 320 with open(temp_json_file, "w", encoding="utf-8") as f: 321 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 322 323 revision_to_update = 369411 324 325 # Index of the tryjob that is going to have its 'status' value 326 # updated. 327 tryjob_index = 0 328 329 custom_script = None 330 331 update_tryjob_status.UpdateTryjobStatus( 332 revision_to_update, 333 update_tryjob_status.TryjobStatus.BAD, 334 temp_json_file, 335 custom_script, 336 ) 337 338 # Verify that the tryjob's 'status' has been updated in the status 339 # file. 340 with open(temp_json_file, encoding="utf-8") as status_file: 341 bisect_contents = json.load(status_file) 342 343 self.assertEqual( 344 bisect_contents["jobs"][tryjob_index]["status"], 345 update_tryjob_status.TryjobStatus.BAD.value, 346 ) 347 348 mock_find_tryjob_index.assert_called_once() 349 350 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 351 # status file. 352 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 353 def testSuccessfullyUpdatedTryjobStatusToPending( 354 self, mock_find_tryjob_index 355 ): 356 bisect_test_contents = { 357 "start": 369410, 358 "end": 369420, 359 "jobs": [{"rev": 369411, "status": "skip"}], 360 } 361 362 # Create a temporary .JSON file to simulate a .JSON file that has 363 # bisection contents. 364 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 365 with open(temp_json_file, "w", encoding="utf-8") as f: 366 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 367 368 revision_to_update = 369411 369 370 # Index of the tryjob that is going to have its 'status' value 371 # updated. 372 tryjob_index = 0 373 374 custom_script = None 375 376 update_tryjob_status.UpdateTryjobStatus( 377 revision_to_update, 378 update_tryjob_status.TryjobStatus.SKIP, 379 temp_json_file, 380 custom_script, 381 ) 382 383 # Verify that the tryjob's 'status' has been updated in the status 384 # file. 385 with open(temp_json_file, encoding="utf-8") as status_file: 386 bisect_contents = json.load(status_file) 387 388 self.assertEqual( 389 bisect_contents["jobs"][tryjob_index]["status"], 390 update_tryjob_status.TryjobStatus.SKIP.value, 391 ) 392 393 mock_find_tryjob_index.assert_called_once() 394 395 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 396 # status file. 397 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 398 def testSuccessfullyUpdatedTryjobStatusToSkip(self, mock_find_tryjob_index): 399 bisect_test_contents = { 400 "start": 369410, 401 "end": 369420, 402 "jobs": [ 403 { 404 "rev": 369411, 405 "status": "pending", 406 } 407 ], 408 } 409 410 # Create a temporary .JSON file to simulate a .JSON file that has 411 # bisection contents. 412 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 413 with open(temp_json_file, "w", encoding="utf-8") as f: 414 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 415 416 revision_to_update = 369411 417 418 # Index of the tryjob that is going to have its 'status' value 419 # updated. 420 tryjob_index = 0 421 422 custom_script = None 423 424 update_tryjob_status.UpdateTryjobStatus( 425 revision_to_update, 426 update_tryjob_status.TryjobStatus.PENDING, 427 temp_json_file, 428 custom_script, 429 ) 430 431 # Verify that the tryjob's 'status' has been updated in the status 432 # file. 433 with open(temp_json_file, encoding="utf-8") as status_file: 434 bisect_contents = json.load(status_file) 435 436 self.assertEqual( 437 bisect_contents["jobs"][tryjob_index]["status"], 438 update_tryjob_status.TryjobStatus.PENDING.value, 439 ) 440 441 mock_find_tryjob_index.assert_called_once() 442 443 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 444 @mock.patch.object( 445 update_tryjob_status, 446 "GetCustomScriptResult", 447 return_value=update_tryjob_status.TryjobStatus.SKIP.value, 448 ) 449 def testUpdatedTryjobStatusToAutoPassedWithCustomScript( 450 self, mock_get_custom_script_result, mock_find_tryjob_index 451 ): 452 bisect_test_contents = { 453 "start": 369410, 454 "end": 369420, 455 "jobs": [ 456 {"rev": 369411, "status": "pending", "buildbucket_id": 1200} 457 ], 458 } 459 460 # Create a temporary .JSON file to simulate a .JSON file that has 461 # bisection contents. 462 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 463 with open(temp_json_file, "w", encoding="utf-8") as f: 464 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 465 466 revision_to_update = 369411 467 468 # Index of the tryjob that is going to have its 'status' value 469 # updated. 470 tryjob_index = 0 471 472 custom_script_path = "/abs/path/to/custom_script.py" 473 474 update_tryjob_status.UpdateTryjobStatus( 475 revision_to_update, 476 update_tryjob_status.TryjobStatus.CUSTOM_SCRIPT, 477 temp_json_file, 478 custom_script_path, 479 ) 480 481 # Verify that the tryjob's 'status' has been updated in the status 482 # file. 483 with open(temp_json_file, encoding="utf-8") as status_file: 484 bisect_contents = json.load(status_file) 485 486 self.assertEqual( 487 bisect_contents["jobs"][tryjob_index]["status"], 488 update_tryjob_status.TryjobStatus.SKIP.value, 489 ) 490 491 mock_get_custom_script_result.assert_called_once() 492 493 mock_find_tryjob_index.assert_called_once() 494 495 # Simulate the behavior of `FindTryjobIndex()` when the tryjob exists in the 496 # status file. 497 @mock.patch.object(update_tryjob_status, "FindTryjobIndex", return_value=0) 498 def testSetStatusDoesNotExistWhenUpdatingTryjobStatus( 499 self, mock_find_tryjob_index 500 ): 501 bisect_test_contents = { 502 "start": 369410, 503 "end": 369420, 504 "jobs": [ 505 {"rev": 369411, "status": "pending", "buildbucket_id": 1200} 506 ], 507 } 508 509 # Create a temporary .JSON file to simulate a .JSON file that has 510 # bisection contents. 511 with test_helpers.CreateTemporaryJsonFile() as temp_json_file: 512 with open(temp_json_file, "w", encoding="utf-8") as f: 513 test_helpers.WritePrettyJsonFile(bisect_test_contents, f) 514 515 revision_to_update = 369411 516 517 nonexistent_update_status = "revert_status" 518 519 custom_script = None 520 521 # Verify the exception is raised when the `set_status` command line 522 # argument does not exist in the mapping. 523 with self.assertRaises(ValueError) as err: 524 update_tryjob_status.UpdateTryjobStatus( 525 revision_to_update, 526 nonexistent_update_status, 527 temp_json_file, 528 custom_script, 529 ) 530 531 self.assertEqual( 532 str(err.exception), 533 'Invalid "set_status" option provided: revert_status', 534 ) 535 536 mock_find_tryjob_index.assert_called_once() 537 538 539if __name__ == "__main__": 540 unittest.main() 541