1#!/usr/bin/env vpython3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import json 9import os 10import sys 11from typing import Any, Dict, Set, Tuple 12import unittest 13 14if sys.version_info[0] == 2: 15 import mock 16else: 17 import unittest.mock as mock 18 19from pyfakefs import fake_filesystem_unittest 20 21from unexpected_passes_common import builders 22from unexpected_passes_common import constants 23from unexpected_passes_common import data_types 24from unexpected_passes_common import unittest_utils 25 26 27class FakeFilesystemTestCaseWithFileCreation(fake_filesystem_unittest.TestCase): 28 def CreateFile(self, *args, **kwargs): 29 # TODO(crbug.com/1156806): Remove this and just use fs.create_file() when 30 # Catapult is updated to a newer version of pyfakefs that is compatible with 31 # Chromium's version. 32 if hasattr(self.fs, 'create_file'): 33 self.fs.create_file(*args, **kwargs) 34 else: 35 self.fs.CreateFile(*args, **kwargs) 36 37 38class GetCiBuildersUnittest(FakeFilesystemTestCaseWithFileCreation): 39 def setUp(self) -> None: 40 self._builders_instance = unittest_utils.GenericBuilders( 41 suite='webgl_conformance') 42 self._isolate_patcher = mock.patch.object( 43 self._builders_instance, 44 'GetIsolateNames', 45 return_value={'telemetry_gpu_integration_test'}) 46 self._isolate_mock = self._isolate_patcher.start() 47 self.addCleanup(self._isolate_patcher.stop) 48 49 def testJsonContentLoaded(self) -> None: 50 """Tests that the correct JSON data is loaded in.""" 51 self.setUpPyfakefs() 52 gpu_json = { 53 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 54 'Android Release (Nexus 5X)': { 55 'isolated_scripts': [{ 56 'args': [ 57 'webgl_conformance', 58 ], 59 'isolate_name': 60 'telemetry_gpu_integration_test', 61 }], 62 }, 63 'GPU Linux Builder': {}, 64 } 65 gpu_fyi_json = { 66 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 67 'ANGLE GPU Android Release (Nexus 5X)': { 68 'isolated_scripts': [{ 69 'args': [ 70 'webgl_conformance', 71 ], 72 'isolate_name': 73 'telemetry_gpu_integration_test', 74 }], 75 }, 76 'GPU FYI Linux Builder': {}, 77 } 78 # Should be ignored. 79 tryserver_json = { 80 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 81 'Trybot': { 82 'isolated_scripts': [{ 83 'args': [ 84 'webgl_conformance', 85 ], 86 'isolate_name': 87 'telemetry_gpu_integration_test', 88 }], 89 }, 90 } 91 # Also should be ignored. 92 not_buildbot_json = { 93 'Not buildbot': { 94 'isolated_scripts': [{ 95 'args': [ 96 'webgl_conformance', 97 ], 98 'isolate_name': 99 'telemetry_gpu_integration_test', 100 }], 101 }, 102 } 103 104 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 105 'chromium.gpu.json'), 106 contents=json.dumps(gpu_json)) 107 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 108 'chromium.gpu.fyi.json'), 109 contents=json.dumps(gpu_fyi_json)) 110 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 111 'tryserver.gpu.json'), 112 contents=json.dumps(tryserver_json)) 113 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 114 'not_buildbot.json'), 115 contents=json.dumps(not_buildbot_json)) 116 117 gpu_builders = self._builders_instance.GetCiBuilders() 118 self.assertEqual( 119 gpu_builders, 120 set([ 121 data_types.BuilderEntry('Android Release (Nexus 5X)', 122 constants.BuilderTypes.CI, False), 123 data_types.BuilderEntry('ANGLE GPU Android Release (Nexus 5X)', 124 constants.BuilderTypes.CI, False), 125 data_types.BuilderEntry('GPU Linux Builder', 126 constants.BuilderTypes.CI, False), 127 data_types.BuilderEntry('GPU FYI Linux Builder', 128 constants.BuilderTypes.CI, False), 129 ])) 130 131 def testPublicInternalBuilders(self) -> None: 132 """Tests that public internal builders are treated as internal.""" 133 self.setUpPyfakefs() 134 gpu_json = { 135 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 136 'Android Release (Nexus 5X)': { 137 'isolated_scripts': [{ 138 'args': [ 139 'webgl_conformance', 140 ], 141 'isolate_name': 142 'telemetry_gpu_integration_test', 143 }], 144 }, 145 'GPU Linux Builder': {}, 146 } 147 gpu_internal_json = { 148 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 149 'Android Chrome Release (Nexus 5X)': { 150 'isolated_scripts': [{ 151 'args': [ 152 'webgl_conformance', 153 ], 154 'isolate_name': 155 'telemetry_gpu_integration_test', 156 }], 157 }, 158 'GPU Chrome Linux Builder': {}, 159 } 160 internal_json = { 161 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 162 'Android Internal Release (Nexus 5X)': { 163 'isolated_scripts': [{ 164 'args': [ 165 'webgl_conformance', 166 ], 167 'isolate_name': 168 'telemetry_gpu_integration_test', 169 }], 170 }, 171 'GPU Internal Linux Builder': {}, 172 } 173 174 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 175 'chromium.gpu.json'), 176 contents=json.dumps(gpu_json)) 177 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 178 'chrome.gpu.fyi.json'), 179 contents=json.dumps(gpu_internal_json)) 180 self.CreateFile(os.path.join(builders.INTERNAL_TESTING_BUILDBOT_DIR, 181 'internal.json'), 182 contents=json.dumps(internal_json)) 183 184 gpu_builders = self._builders_instance.GetCiBuilders() 185 self.assertEqual( 186 gpu_builders, 187 set([ 188 data_types.BuilderEntry('Android Release (Nexus 5X)', 189 constants.BuilderTypes.CI, False), 190 data_types.BuilderEntry('GPU Linux Builder', 191 constants.BuilderTypes.CI, False), 192 ])) 193 194 internal_instance = unittest_utils.GenericBuilders( 195 suite='webgl_conformance', include_internal_builders=True) 196 with mock.patch.object(internal_instance, 197 'GetIsolateNames', 198 return_value={'telemetry_gpu_integration_test'}): 199 gpu_builders = internal_instance.GetCiBuilders() 200 self.assertEqual( 201 gpu_builders, 202 set([ 203 data_types.BuilderEntry('Android Release (Nexus 5X)', 204 constants.BuilderTypes.CI, False), 205 data_types.BuilderEntry('Android Chrome Release (Nexus 5X)', 206 constants.BuilderTypes.CI, True), 207 data_types.BuilderEntry('Android Internal Release (Nexus 5X)', 208 constants.BuilderTypes.CI, True), 209 data_types.BuilderEntry('GPU Linux Builder', 210 constants.BuilderTypes.CI, False), 211 data_types.BuilderEntry('GPU Chrome Linux Builder', 212 constants.BuilderTypes.CI, True), 213 data_types.BuilderEntry('GPU Internal Linux Builder', 214 constants.BuilderTypes.CI, True), 215 ])) 216 217 def testFilterBySuite(self) -> None: 218 """Tests that only builders that run the given suite are returned.""" 219 220 def SideEffect(tm: Dict[str, Any]) -> bool: 221 tests = tm.get('isolated_scripts', []) 222 for t in tests: 223 if t.get('isolate_name') == 'foo_integration_test': 224 if 'webgl_conformance' in t.get('args', []): 225 return True 226 return False 227 228 self.setUpPyfakefs() 229 gpu_json = { 230 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 231 'Android Tester': { 232 'isolated_scripts': [ 233 { 234 'args': [ 235 'webgl_conformance', 236 ], 237 'isolate_name': 'not_telemetry', 238 }, 239 ], 240 }, 241 'Linux Tester': { 242 'isolated_scripts': [ 243 { 244 'args': [ 245 'not_a_suite', 246 ], 247 'isolate_name': 'foo_integration_test', 248 }, 249 ], 250 }, 251 'Windows Tester': { 252 'isolated_scripts': [ 253 { 254 'args': [ 255 'webgl_conformance', 256 ], 257 'isolate_name': 'foo_integration_test', 258 }, 259 ], 260 }, 261 } 262 263 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 264 'chromium.json'), 265 contents=json.dumps(gpu_json)) 266 267 with mock.patch.object(self._builders_instance, 268 '_BuilderRunsTestOfInterest', 269 side_effect=SideEffect): 270 gpu_builders = self._builders_instance.GetCiBuilders() 271 self.assertEqual( 272 gpu_builders, 273 set([ 274 data_types.BuilderEntry('Windows Tester', constants.BuilderTypes.CI, 275 False) 276 ])) 277 278 def testRealContentCanBeLoaded(self) -> None: 279 """Tests that *something* from the real JSON files can be loaded.""" 280 # This directory is not available on swarming, so if it doesn't exist, just 281 # skip the test. 282 if not os.path.exists(builders.TESTING_BUILDBOT_DIR): 283 return 284 self.assertNotEqual(len(self._builders_instance.GetCiBuilders()), 0) 285 286 287class GetMirroredBuildersForCiBuilderUnittest(unittest.TestCase): 288 def setUp(self) -> None: 289 self._builders_instance = builders.Builders('suite', False) 290 self._bb_patcher = mock.patch.object(self._builders_instance, 291 '_GetBuildbucketOutputForCiBuilder') 292 self._bb_mock = self._bb_patcher.start() 293 self.addCleanup(self._bb_patcher.stop) 294 self._fake_ci_patcher = mock.patch.object(self._builders_instance, 295 'GetFakeCiBuilders', 296 return_value={}) 297 self._fake_ci_mock = self._fake_ci_patcher.start() 298 self.addCleanup(self._fake_ci_patcher.stop) 299 self._non_chromium_patcher = mock.patch.object( 300 self._builders_instance, 301 'GetNonChromiumBuilders', 302 return_value={'foo_non_chromium'}) 303 self._non_chromium_mock = self._non_chromium_patcher.start() 304 self.addCleanup(self._non_chromium_patcher.stop) 305 306 def testFakeCiBuilder(self) -> None: 307 """Tests that a fake CI builder gets properly mapped.""" 308 self._fake_ci_mock.return_value = { 309 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, False): 310 {data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, False)} 311 } 312 try_builder, found_mirror = ( 313 self._builders_instance._GetMirroredBuildersForCiBuilder( 314 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 315 False))) 316 self.assertTrue(found_mirror) 317 self.assertEqual( 318 try_builder, 319 set([ 320 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 321 False) 322 ])) 323 self._bb_mock.assert_not_called() 324 325 def testNoBuildbucketOutput(self) -> None: 326 """Tests that a failure to get Buildbucket output is surfaced.""" 327 self._bb_mock.return_value = '' 328 builder_entry = data_types.BuilderEntry('nonexistent', 329 constants.BuilderTypes.CI, False) 330 try_builder, found_mirror = ( 331 self._builders_instance._GetMirroredBuildersForCiBuilder(builder_entry)) 332 self.assertFalse(found_mirror) 333 self.assertEqual(try_builder, set([builder_entry])) 334 335 def testBuildbucketOutput(self): 336 """Tests that Buildbucket output is parsed correctly.""" 337 self._bb_mock.return_value = json.dumps({ 338 'output': { 339 'properties': { 340 'mirrored_builders': [ 341 'try:foo_try', 342 'try:bar_try', 343 ] 344 } 345 } 346 }) 347 try_builders, found_mirror = ( 348 self._builders_instance._GetMirroredBuildersForCiBuilder( 349 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 350 False))) 351 self.assertTrue(found_mirror) 352 self.assertEqual( 353 try_builders, 354 set([ 355 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 356 False), 357 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, 358 False) 359 ])) 360 361 def testBuildbucketOutputInternal(self) -> None: 362 """Tests that internal Buildbucket output is parsed correctly.""" 363 self._bb_mock.return_value = json.dumps({ 364 'output': { 365 'properties': { 366 'mirrored_builders': [ 367 'try:foo_try', 368 'try:bar_try', 369 ] 370 } 371 } 372 }) 373 try_builders, found_mirror = ( 374 self._builders_instance._GetMirroredBuildersForCiBuilder( 375 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, True))) 376 self.assertTrue(found_mirror) 377 self.assertEqual( 378 try_builders, 379 set([ 380 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 381 True), 382 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, True) 383 ])) 384 385 386class GetTryBuildersUnittest(FakeFilesystemTestCaseWithFileCreation): 387 def setUp(self) -> None: 388 self._builders_instance = builders.Builders('suite', False) 389 self._get_patcher = mock.patch.object(self._builders_instance, 390 '_GetMirroredBuildersForCiBuilder') 391 self._get_mock = self._get_patcher.start() 392 self.addCleanup(self._get_patcher.stop) 393 self._runs_test_patcher = mock.patch.object(self._builders_instance, 394 '_BuilderRunsTestOfInterest') 395 self._runs_test_mock = self._runs_test_patcher.start() 396 self.addCleanup(self._runs_test_patcher.stop) 397 398 self.setUpPyfakefs() 399 # Make sure the directory exists. 400 self.CreateFile( 401 os.path.join(builders.TESTING_BUILDBOT_DIR, 'placeholder.txt')) 402 403 def testMirrorNoOutputCausesFailure(self) -> None: 404 """Tests that a failure to get Buildbot output raises an exception.""" 405 builder = data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, 406 False) 407 self._get_mock.return_value = (set([builder]), False) 408 self._runs_test_mock.return_value = True 409 with self.assertRaises(RuntimeError): 410 self._builders_instance.GetTryBuilders([builder]) 411 412 def testMirrorOutputReturned(self) -> None: 413 """Tests that parsed, mirrored builders get returned on success.""" 414 415 def SideEffect(ci_builder: data_types.BuilderEntry 416 ) -> Tuple[Set[data_types.BuilderEntry], bool]: 417 b = [ 418 data_types.BuilderEntry(ci_builder.name.replace('ci', 'try'), 419 constants.BuilderTypes.TRY, False), 420 data_types.BuilderEntry(ci_builder.name.replace('ci', 'try2'), 421 constants.BuilderTypes.TRY, False), 422 ] 423 return set(b), True 424 425 self._get_mock.side_effect = SideEffect 426 self._runs_test_mock.return_value = False 427 mirrored_builders = self._builders_instance.GetTryBuilders([ 428 data_types.BuilderEntry('foo_ci', constants.BuilderTypes.CI, False), 429 data_types.BuilderEntry('bar_ci', constants.BuilderTypes.CI, False), 430 ]) 431 self.assertEqual( 432 mirrored_builders, 433 set([ 434 data_types.BuilderEntry('foo_try', constants.BuilderTypes.TRY, 435 False), 436 data_types.BuilderEntry('foo_try2', constants.BuilderTypes.TRY, 437 False), 438 data_types.BuilderEntry('bar_try', constants.BuilderTypes.TRY, 439 False), 440 data_types.BuilderEntry('bar_try2', constants.BuilderTypes.TRY, 441 False), 442 ])) 443 444 def testDedicatedJsonContentLoaded(self) -> None: 445 """Tests that tryserver JSON content is loaded.""" 446 447 def SideEffect(test_spec: Dict[str, Any]) -> bool: 448 # Treat non-empty test specs as valid. 449 return bool(test_spec) 450 451 self._runs_test_mock.side_effect = SideEffect 452 # Should be ignored. 453 gpu_json = { 454 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 455 'Android Release (Nexus 5X)': { 456 'isolated_scripts': [{ 457 'args': [ 458 'webgl_conformance', 459 ], 460 'isolate_name': 461 'telemetry_gpu_integration_test', 462 }], 463 }, 464 'GPU Linux Builder': {}, 465 } 466 # Should be ignored. 467 gpu_fyi_json = { 468 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 469 'ANGLE GPU Android Release (Nexus 5X)': { 470 'isolated_scripts': [{ 471 'args': [ 472 'webgl_conformance', 473 ], 474 'isolate_name': 475 'telemetry_gpu_integration_test', 476 }], 477 }, 478 'GPU FYI Linux Builder': {}, 479 } 480 tryserver_json = { 481 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 482 'Trybot': { 483 'isolated_scripts': [{ 484 'args': [ 485 'webgl_conformance', 486 ], 487 'isolate_name': 488 'telemetry_gpu_integration_test', 489 }], 490 }, 491 'Trybot Empty': {}, 492 } 493 # Also should be ignored. 494 not_buildbot_json = { 495 'Not buildbot': { 496 'isolated_scripts': [{ 497 'args': [ 498 'webgl_conformance', 499 ], 500 'isolate_name': 501 'telemetry_gpu_integration_test', 502 }], 503 }, 504 } 505 506 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 507 'chromium.gpu.json'), 508 contents=json.dumps(gpu_json)) 509 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 510 'chromium.gpu.fyi.json'), 511 contents=json.dumps(gpu_fyi_json)) 512 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 513 'tryserver.gpu.json'), 514 contents=json.dumps(tryserver_json)) 515 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 516 'not_buildbot.json'), 517 contents=json.dumps(not_buildbot_json)) 518 519 gpu_builders = self._builders_instance.GetTryBuilders({}) 520 self.assertEqual( 521 gpu_builders, 522 set([ 523 data_types.BuilderEntry('Trybot', constants.BuilderTypes.TRY, 524 False), 525 ])) 526 527 def testDedicatedFilterBySuite(self) -> None: 528 """Tests that only builders that run the given suite are returned.""" 529 530 def SideEffect(tm: Dict[str, Any]) -> bool: 531 tests = tm.get('isolated_scripts', []) 532 for t in tests: 533 if t.get('isolate_name') == 'foo_integration_test': 534 if 'webgl_conformance' in t.get('args', []): 535 return True 536 return False 537 538 self._runs_test_mock.side_effect = SideEffect 539 gpu_json = { 540 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 541 'Android Tester': { 542 'isolated_scripts': [ 543 { 544 'args': [ 545 'webgl_conformance', 546 ], 547 'isolate_name': 'not_telemetry', 548 }, 549 ], 550 }, 551 'Linux Tester': { 552 'isolated_scripts': [ 553 { 554 'args': [ 555 'not_a_suite', 556 ], 557 'isolate_name': 'foo_integration_test', 558 }, 559 ], 560 }, 561 'Windows Tester': { 562 'isolated_scripts': [ 563 { 564 'args': [ 565 'webgl_conformance', 566 ], 567 'isolate_name': 'foo_integration_test', 568 }, 569 ], 570 }, 571 } 572 573 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 574 'tryserver.chromium.json'), 575 contents=json.dumps(gpu_json)) 576 577 gpu_builders = self._builders_instance.GetTryBuilders({}) 578 self.assertEqual( 579 gpu_builders, 580 set([ 581 data_types.BuilderEntry('Windows Tester', 582 constants.BuilderTypes.TRY, False) 583 ])) 584 585 def testDedicatedAndMirroredCombined(self) -> None: 586 """Tests that both dedicated and mirrored trybots are returned.""" 587 588 def SideEffect(_: Any) -> Tuple[Set[data_types.BuilderEntry], bool]: 589 return set({ 590 data_types.BuilderEntry('mirrored_trybot', constants.BuilderTypes.TRY, 591 False) 592 }), True 593 594 self._get_mock.side_effect = SideEffect 595 self._runs_test_mock.return_value = True 596 tryserver_json = { 597 'AAAAA1 AUTOGENERATED FILE DO NOT EDIT': {}, 598 'Trybot': { 599 'isolated_scripts': [{ 600 'args': [ 601 'webgl_conformance', 602 ], 603 'isolate_name': 604 'telemetry_gpu_integration_test', 605 }], 606 }, 607 } 608 609 self.CreateFile(os.path.join(builders.TESTING_BUILDBOT_DIR, 610 'tryserver.chromium.json'), 611 contents=json.dumps(tryserver_json)) 612 613 try_builders = self._builders_instance.GetTryBuilders({ 614 data_types.BuilderEntry('ci_builder', constants.BuilderTypes.CI, False) 615 }) 616 self.assertEqual( 617 try_builders, { 618 data_types.BuilderEntry('mirrored_trybot', 619 constants.BuilderTypes.TRY, False), 620 data_types.BuilderEntry('Trybot', constants.BuilderTypes.TRY, False) 621 }) 622 623 624if __name__ == '__main__': 625 unittest.main(verbosity=2) 626