1#!/usr/bin/env vpython3 2# Copyright 2020 The Chromium Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from __future__ import print_function 7 8import datetime 9import os 10import sys 11import tempfile 12import unittest 13 14if sys.version_info[0] == 2: 15 import mock 16else: 17 import unittest.mock as mock 18 19from pyfakefs import fake_filesystem_unittest 20 21from unexpected_passes_common import data_types 22from unexpected_passes_common import expectations 23from unexpected_passes_common import unittest_utils as uu 24 25FAKE_EXPECTATION_FILE_CONTENTS = """\ 26# tags: [ win linux ] 27# results: [ Failure RetryOnFailure Skip Pass ] 28crbug.com/1234 [ win ] foo/test [ Failure ] 29crbug.com/5678 crbug.com/6789 [ win ] foo/another/test [ RetryOnFailure ] 30 31[ linux ] foo/test [ Failure ] 32 33crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ] 34crbug.com/3456 [ linux ] some/bad/test [ Skip ] 35crbug.com/4567 [ linux ] some/good/test [ Pass ] 36""" 37 38SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS = """\ 39# tags: [ mac ] 40# results: [ Failure ] 41 42crbug.com/4567 [ mac ] foo/test [ Failure ] 43""" 44 45FAKE_EXPECTATION_FILE_CONTENTS_WITH_TYPO = """\ 46# tags: [ win linux ] 47# results: [ Failure RetryOnFailure Skip ] 48crbug.com/1234 [ wine ] foo/test [ Failure ] 49 50[ linux ] foo/test [ Failure ] 51 52crbug.com/2345 [ linux ] bar/* [ RetryOnFailure ] 53crbug.com/3456 [ linux ] some/bad/test [ Skip ] 54""" 55 56FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS = """\ 57# tags: [ win win10 58# linux 59# mac ] 60# tags: [ nvidia nvidia-0x1111 61# intel intel-0x2222 62# amd amd-0x3333] 63# tags: [ release debug ] 64# results: [ Failure RetryOnFailure ] 65 66crbug.com/1234 [ win ] foo/test [ Failure ] 67crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 68""" 69 70 71class CreateTestExpectationMapUnittest(unittest.TestCase): 72 def setUp(self) -> None: 73 self.instance = expectations.Expectations() 74 75 self._expectation_content = {} 76 self._content_patcher = mock.patch.object( 77 self.instance, '_GetNonRecentExpectationContent') 78 self._content_mock = self._content_patcher.start() 79 self.addCleanup(self._content_patcher.stop) 80 81 def SideEffect(filepath, _): 82 return self._expectation_content[filepath] 83 84 self._content_mock.side_effect = SideEffect 85 86 def testExclusiveOr(self) -> None: 87 """Tests that only one input can be specified.""" 88 with self.assertRaises(AssertionError): 89 self.instance.CreateTestExpectationMap(None, None, 90 datetime.timedelta(days=0)) 91 with self.assertRaises(AssertionError): 92 self.instance.CreateTestExpectationMap('foo', ['bar'], 93 datetime.timedelta(days=0)) 94 95 def testExpectationFile(self) -> None: 96 """Tests reading expectations from an expectation file.""" 97 filename = '/tmp/foo' 98 self._expectation_content[filename] = FAKE_EXPECTATION_FILE_CONTENTS 99 expectation_map = self.instance.CreateTestExpectationMap( 100 filename, None, datetime.timedelta(days=0)) 101 # Skip expectations should be omitted, but everything else should be 102 # present. 103 # yapf: disable 104 expected_expectation_map = { 105 filename: { 106 data_types.Expectation( 107 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {}, 108 data_types.Expectation( 109 'foo/another/test', ['win'], ['RetryOnFailure'], 110 'crbug.com/5678 crbug.com/6789'): {}, 111 data_types.Expectation('foo/test', ['linux'], ['Failure']): {}, 112 data_types.Expectation( 113 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {}, 114 }, 115 } 116 # yapf: enable 117 self.assertEqual(expectation_map, expected_expectation_map) 118 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 119 120 def testMultipleExpectationFiles(self) -> None: 121 """Tests reading expectations from multiple files.""" 122 filename1 = '/tmp/foo' 123 filename2 = '/tmp/bar' 124 expectation_files = [filename1, filename2] 125 self._expectation_content[filename1] = FAKE_EXPECTATION_FILE_CONTENTS 126 self._expectation_content[ 127 filename2] = SECONDARY_FAKE_EXPECTATION_FILE_CONTENTS 128 129 expectation_map = self.instance.CreateTestExpectationMap( 130 expectation_files, None, datetime.timedelta(days=0)) 131 # yapf: disable 132 expected_expectation_map = { 133 expectation_files[0]: { 134 data_types.Expectation( 135 'foo/test', ['win'], ['Failure'], 'crbug.com/1234'): {}, 136 data_types.Expectation( 137 'foo/another/test', ['win'], ['RetryOnFailure'], 138 'crbug.com/5678 crbug.com/6789'): {}, 139 data_types.Expectation('foo/test', ['linux'], ['Failure']): {}, 140 data_types.Expectation( 141 'bar/*', ['linux'], ['RetryOnFailure'], 'crbug.com/2345'): {}, 142 }, 143 expectation_files[1]: { 144 data_types.Expectation( 145 'foo/test', ['mac'], ['Failure'], 'crbug.com/4567'): {}, 146 } 147 } 148 # yapf: enable 149 self.assertEqual(expectation_map, expected_expectation_map) 150 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 151 152 def testIndividualTests(self) -> None: 153 """Tests reading expectations from a list of tests.""" 154 expectation_map = self.instance.CreateTestExpectationMap( 155 None, ['foo/test', 'bar/*'], datetime.timedelta(days=0)) 156 expected_expectation_map = { 157 '': { 158 data_types.Expectation('foo/test', [], ['RetryOnFailure']): {}, 159 data_types.Expectation('bar/*', [], ['RetryOnFailure']): {}, 160 }, 161 } 162 self.assertEqual(expectation_map, expected_expectation_map) 163 self.assertIsInstance(expectation_map, data_types.TestExpectationMap) 164 165 166class GetNonRecentExpectationContentUnittest(unittest.TestCase): 167 def setUp(self) -> None: 168 self.instance = uu.CreateGenericExpectations() 169 self._output_patcher = mock.patch( 170 'unexpected_passes_common.expectations.subprocess.check_output') 171 self._output_mock = self._output_patcher.start() 172 self.addCleanup(self._output_patcher.stop) 173 174 def testBasic(self) -> None: 175 """Tests that only expectations that are old enough are kept.""" 176 today_date = datetime.date.today() 177 yesterday_date = today_date - datetime.timedelta(days=1) 178 older_date = today_date - datetime.timedelta(days=2) 179 today_str = today_date.isoformat() 180 yesterday_str = yesterday_date.isoformat() 181 older_str = older_date.isoformat() 182 # pylint: disable=line-too-long 183 blame_output = """\ 1845f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ] 18598637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ] 1863fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ] 1875f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4) 1885f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ] 18998637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment 1903fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ] 1915f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 8)crbug.com/2345 testname [ Failure ] 1923fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 9)crbug.com/3456 othertest [ Failure ]""" 193 # pylint: enable=line-too-long 194 blame_output = blame_output.format(today_date=today_str, 195 yesterday_date=yesterday_str, 196 older_date=older_str) 197 self._output_mock.return_value = blame_output.encode('utf-8') 198 199 expected_content = """\ 200# tags: [ tag1 ] 201# tags: [ tag2 ] 202# results: [ Failure ] 203 204[ tag1 ] othertest [ Failure ] 205crbug.com/3456 othertest [ Failure ]""" 206 self.assertEqual( 207 self.instance._GetNonRecentExpectationContent( 208 '', datetime.timedelta(days=1)), expected_content) 209 210 def testNegativeGracePeriod(self) -> None: 211 """Tests that setting a negative grace period disables filtering.""" 212 today_date = datetime.date.today() 213 yesterday_date = today_date - datetime.timedelta(days=1) 214 older_date = today_date - datetime.timedelta(days=2) 215 today_str = today_date.isoformat() 216 yesterday_str = yesterday_date.isoformat() 217 older_str = older_date.isoformat() 218 # pylint: disable=line-too-long 219 blame_output = """\ 2205f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 1)# tags: [ tag1 ] 22198637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 2)# tags: [ tag2 ] 2223fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 3)# results: [ Failure ] 2235f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 4) 2245f03bc04975c04 (Some R. Author {today_date} 00:00:00 +0000 5)crbug.com/1234 [ tag1 ] testname [ Failure ] 22598637cd80f8c15 (Some R. Author {yesterday_date} 00:00:00 +0000 6)[ tag2 ] testname [ Failure ] # Comment 2263fcadac9d861d0 (Some R. Author {older_date} 00:00:00 +0000 7)[ tag1 ] othertest [ Failure ]""" 227 # pylint: enable=line-too-long 228 blame_output = blame_output.format(today_date=today_str, 229 yesterday_date=yesterday_str, 230 older_date=older_str) 231 self._output_mock.return_value = blame_output.encode('utf-8') 232 233 expected_content = """\ 234# tags: [ tag1 ] 235# tags: [ tag2 ] 236# results: [ Failure ] 237 238crbug.com/1234 [ tag1 ] testname [ Failure ] 239[ tag2 ] testname [ Failure ] # Comment 240[ tag1 ] othertest [ Failure ]""" 241 self.assertEqual( 242 self.instance._GetNonRecentExpectationContent( 243 '', datetime.timedelta(days=-1)), expected_content) 244 245 246class RemoveExpectationsFromFileUnittest(fake_filesystem_unittest.TestCase): 247 def setUp(self) -> None: 248 self.instance = uu.CreateGenericExpectations() 249 self.header = self.instance._GetExpectationFileTagHeader(None) 250 self.setUpPyfakefs() 251 with tempfile.NamedTemporaryFile(delete=False) as f: 252 self.filename = f.name 253 254 def testExpectationRemoval(self) -> None: 255 """Tests that expectations are properly removed from a file.""" 256 contents = self.header + """ 257 258# This is a test comment 259crbug.com/1234 [ win ] foo/test [ Failure ] 260crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 261 262# Another comment 263[ linux ] bar/test [ RetryOnFailure ] 264[ win ] bar/test [ RetryOnFailure ] 265""" 266 267 stale_expectations = [ 268 data_types.Expectation('foo/test', ['win'], ['Failure'], 269 'crbug.com/1234'), 270 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 271 ] 272 273 expected_contents = self.header + """ 274 275# This is a test comment 276crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 277 278# Another comment 279[ win ] bar/test [ RetryOnFailure ] 280""" 281 282 with open(self.filename, 'w') as f: 283 f.write(contents) 284 285 removed_urls = self.instance.RemoveExpectationsFromFile( 286 stale_expectations, self.filename, expectations.RemovalType.STALE) 287 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 288 with open(self.filename) as f: 289 self.assertEqual(f.read(), expected_contents) 290 291 def testRemovalWithMultipleBugs(self) -> None: 292 """Tests removal of expectations with multiple associated bugs.""" 293 contents = self.header + """ 294 295# This is a test comment 296crbug.com/1234 crbug.com/3456 crbug.com/4567 [ win ] foo/test [ Failure ] 297crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 298 299# Another comment 300[ linux ] bar/test [ RetryOnFailure ] 301[ win ] bar/test [ RetryOnFailure ] 302""" 303 304 stale_expectations = [ 305 data_types.Expectation('foo/test', ['win'], ['Failure'], 306 'crbug.com/1234 crbug.com/3456 crbug.com/4567'), 307 ] 308 expected_contents = self.header + """ 309 310# This is a test comment 311crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 312 313# Another comment 314[ linux ] bar/test [ RetryOnFailure ] 315[ win ] bar/test [ RetryOnFailure ] 316""" 317 with open(self.filename, 'w') as f: 318 f.write(contents) 319 320 removed_urls = self.instance.RemoveExpectationsFromFile( 321 stale_expectations, self.filename, expectations.RemovalType.STALE) 322 self.assertEqual( 323 removed_urls, 324 set(['crbug.com/1234', 'crbug.com/3456', 'crbug.com/4567'])) 325 with open(self.filename) as f: 326 self.assertEqual(f.read(), expected_contents) 327 328 def testGeneralBlockComments(self) -> None: 329 """Tests that expectations in a disable block comment are not removed.""" 330 contents = self.header + """ 331crbug.com/1234 [ win ] foo/test [ Failure ] 332# finder:disable-general 333crbug.com/2345 [ win ] foo/test [ Failure ] 334crbug.com/3456 [ win ] foo/test [ Failure ] 335# finder:enable-general 336crbug.com/4567 [ win ] foo/test [ Failure ] 337""" 338 stale_expectations = [ 339 data_types.Expectation('foo/test', ['win'], ['Failure'], 340 'crbug.com/1234'), 341 data_types.Expectation('foo/test', ['win'], ['Failure'], 342 'crbug.com/2345'), 343 data_types.Expectation('foo/test', ['win'], ['Failure'], 344 'crbug.com/3456'), 345 data_types.Expectation('foo/test', ['win'], ['Failure'], 346 'crbug.com/4567'), 347 ] 348 expected_contents = self.header + """ 349# finder:disable-general 350crbug.com/2345 [ win ] foo/test [ Failure ] 351crbug.com/3456 [ win ] foo/test [ Failure ] 352# finder:enable-general 353""" 354 for removal_type in (expectations.RemovalType.STALE, 355 expectations.RemovalType.UNUSED): 356 with open(self.filename, 'w') as f: 357 f.write(contents) 358 removed_urls = self.instance.RemoveExpectationsFromFile( 359 stale_expectations, self.filename, removal_type) 360 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/4567'])) 361 with open(self.filename) as f: 362 self.assertEqual(f.read(), expected_contents) 363 364 def testStaleBlockComments(self) -> None: 365 """Tests that stale expectations in a stale disable block are not removed""" 366 contents = self.header + """ 367crbug.com/1234 [ win ] not_stale [ Failure ] 368crbug.com/1234 [ win ] before_block [ Failure ] 369# finder:disable-stale 370crbug.com/2345 [ win ] in_block [ Failure ] 371# finder:enable-stale 372crbug.com/3456 [ win ] after_block [ Failure ] 373""" 374 stale_expectations = [ 375 data_types.Expectation('before_block', ['win'], 'Failure', 376 'crbug.com/1234'), 377 data_types.Expectation('in_block', ['win'], 'Failure', 378 'crbug.com/2345'), 379 data_types.Expectation('after_block', ['win'], 'Failure', 380 'crbug.com/3456'), 381 ] 382 expected_contents = self.header + """ 383crbug.com/1234 [ win ] not_stale [ Failure ] 384# finder:disable-stale 385crbug.com/2345 [ win ] in_block [ Failure ] 386# finder:enable-stale 387""" 388 with open(self.filename, 'w') as f: 389 f.write(contents) 390 removed_urls = self.instance.RemoveExpectationsFromFile( 391 stale_expectations, self.filename, expectations.RemovalType.STALE) 392 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 393 with open(self.filename) as f: 394 self.assertEqual(f.read(), expected_contents) 395 396 def testUnusedBlockComments(self) -> None: 397 """Tests that stale expectations in unused disable blocks are not removed""" 398 contents = self.header + """ 399crbug.com/1234 [ win ] not_unused [ Failure ] 400crbug.com/1234 [ win ] before_block [ Failure ] 401# finder:disable-unused 402crbug.com/2345 [ win ] in_block [ Failure ] 403# finder:enable-unused 404crbug.com/3456 [ win ] after_block [ Failure ] 405""" 406 unused_expectations = [ 407 data_types.Expectation('before_block', ['win'], 'Failure', 408 'crbug.com/1234'), 409 data_types.Expectation('in_block', ['win'], 'Failure', 410 'crbug.com/2345'), 411 data_types.Expectation('after_block', ['win'], 'Failure', 412 'crbug.com/3456'), 413 ] 414 expected_contents = self.header + """ 415crbug.com/1234 [ win ] not_unused [ Failure ] 416# finder:disable-unused 417crbug.com/2345 [ win ] in_block [ Failure ] 418# finder:enable-unused 419""" 420 with open(self.filename, 'w') as f: 421 f.write(contents) 422 removed_urls = self.instance.RemoveExpectationsFromFile( 423 unused_expectations, self.filename, expectations.RemovalType.UNUSED) 424 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 425 with open(self.filename) as f: 426 self.assertEqual(f.read(), expected_contents) 427 428 def testMismatchedBlockComments(self) -> None: 429 """Tests that block comments for the wrong removal type do nothing.""" 430 contents = self.header + """ 431crbug.com/1234 [ win ] do_not_remove [ Failure ] 432# finder:disable-stale 433crbug.com/2345 [ win ] disabled_stale [ Failure ] 434# finder:enable-stale 435# finder:disable-unused 436crbug.com/3456 [ win ] disabled_unused [ Failure ] 437# finder:enable-unused 438crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 439""" 440 expectations_to_remove = [ 441 data_types.Expectation('disabled_stale', ['win'], 'Failure', 442 'crbug.com/2345'), 443 data_types.Expectation('disabled_unused', ['win'], 'Failure', 444 'crbug.com/3456'), 445 ] 446 447 expected_contents = self.header + """ 448crbug.com/1234 [ win ] do_not_remove [ Failure ] 449# finder:disable-stale 450crbug.com/2345 [ win ] disabled_stale [ Failure ] 451# finder:enable-stale 452crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 453""" 454 with open(self.filename, 'w') as f: 455 f.write(contents) 456 removed_urls = self.instance.RemoveExpectationsFromFile( 457 expectations_to_remove, self.filename, expectations.RemovalType.STALE) 458 self.assertEqual(removed_urls, set(['crbug.com/3456'])) 459 with open(self.filename) as f: 460 self.assertEqual(f.read(), expected_contents) 461 462 expected_contents = self.header + """ 463crbug.com/1234 [ win ] do_not_remove [ Failure ] 464# finder:disable-unused 465crbug.com/3456 [ win ] disabled_unused [ Failure ] 466# finder:enable-unused 467crbug.com/4567 [ win ] also_do_not_remove [ Failure ] 468""" 469 with open(self.filename, 'w') as f: 470 f.write(contents) 471 removed_urls = self.instance.RemoveExpectationsFromFile( 472 expectations_to_remove, self.filename, expectations.RemovalType.UNUSED) 473 self.assertEqual(removed_urls, set(['crbug.com/2345'])) 474 with open(self.filename) as f: 475 self.assertEqual(f.read(), expected_contents) 476 477 def testInlineGeneralComments(self) -> None: 478 """Tests that expectations with inline disable comments are not removed.""" 479 contents = self.header + """ 480crbug.com/1234 [ win ] foo/test [ Failure ] 481crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general 482crbug.com/3456 [ win ] foo/test [ Failure ] 483""" 484 stale_expectations = [ 485 data_types.Expectation('foo/test', ['win'], ['Failure'], 486 'crbug.com/1234'), 487 data_types.Expectation('foo/test', ['win'], ['Failure'], 488 'crbug.com/2345'), 489 data_types.Expectation('foo/test', ['win'], ['Failure'], 490 'crbug.com/3456'), 491 ] 492 expected_contents = self.header + """ 493crbug.com/2345 [ win ] foo/test [ Failure ] # finder:disable-general 494""" 495 for removal_type in (expectations.RemovalType.STALE, 496 expectations.RemovalType.UNUSED): 497 with open(self.filename, 'w') as f: 498 f.write(contents) 499 removed_urls = self.instance.RemoveExpectationsFromFile( 500 stale_expectations, self.filename, removal_type) 501 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 502 with open(self.filename) as f: 503 self.assertEqual(f.read(), expected_contents) 504 505 def testInlineStaleComments(self) -> None: 506 """Tests that expectations with inline stale disable comments not removed""" 507 contents = self.header + """ 508crbug.com/1234 [ win ] not_disabled [ Failure ] 509crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 510crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 511""" 512 stale_expectations = [ 513 data_types.Expectation('not_disabled', ['win'], 'Failure', 514 'crbug.com/1234'), 515 data_types.Expectation('stale_disabled', ['win'], 'Failure', 516 'crbug.com/2345'), 517 data_types.Expectation('unused_disabled', ['win'], 'Failure', 518 'crbug.com/3456') 519 ] 520 expected_contents = self.header + """ 521crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 522""" 523 with open(self.filename, 'w') as f: 524 f.write(contents) 525 removed_urls = self.instance.RemoveExpectationsFromFile( 526 stale_expectations, self.filename, expectations.RemovalType.STALE) 527 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/3456'])) 528 with open(self.filename) as f: 529 self.assertEqual(f.read(), expected_contents) 530 531 def testInlineUnusedComments(self) -> None: 532 """Tests that expectations with inline unused comments not removed""" 533 contents = self.header + """ 534crbug.com/1234 [ win ] not_disabled [ Failure ] 535crbug.com/2345 [ win ] stale_disabled [ Failure ] # finder:disable-stale 536crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 537""" 538 stale_expectations = [ 539 data_types.Expectation('not_disabled', ['win'], 'Failure', 540 'crbug.com/1234'), 541 data_types.Expectation('stale_disabled', ['win'], 'Failure', 542 'crbug.com/2345'), 543 data_types.Expectation('unused_disabled', ['win'], 'Failure', 544 'crbug.com/3456') 545 ] 546 expected_contents = self.header + """ 547crbug.com/3456 [ win ] unused_disabled [ Failure ] # finder:disable-unused 548""" 549 with open(self.filename, 'w') as f: 550 f.write(contents) 551 removed_urls = self.instance.RemoveExpectationsFromFile( 552 stale_expectations, self.filename, expectations.RemovalType.UNUSED) 553 self.assertEqual(removed_urls, set(['crbug.com/1234', 'crbug.com/2345'])) 554 with open(self.filename) as f: 555 self.assertEqual(f.read(), expected_contents) 556 557 def testGetDisableReasonFromComment(self): 558 """Tests that the disable reason can be pulled from a line.""" 559 self.assertEqual( 560 expectations._GetDisableReasonFromComment( 561 '# finder:disable-general foo'), 'foo') 562 self.assertEqual( 563 expectations._GetDisableReasonFromComment( 564 'crbug.com/1234 [ win ] bar/test [ Failure ] ' 565 '# finder:disable-general foo'), 'foo') 566 567 def testGroupBlockAllRemovable(self): 568 """Tests that a group with all members removable is removed.""" 569 contents = self.header + """ 570 571# This is a test comment 572crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 573 574# Another comment 575 576# finder:group-start some group name 577[ linux ] bar/test [ RetryOnFailure ] 578crbug.com/1234 [ win ] foo/test [ Failure ] 579# finder:group-end 580[ win ] bar/test [ RetryOnFailure ] 581""" 582 583 stale_expectations = [ 584 data_types.Expectation('foo/test', ['win'], ['Failure'], 585 'crbug.com/1234'), 586 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']), 587 ] 588 589 expected_contents = self.header + """ 590 591# This is a test comment 592crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 593 594# Another comment 595 596[ win ] bar/test [ RetryOnFailure ] 597""" 598 599 with open(self.filename, 'w') as f: 600 f.write(contents) 601 602 removed_urls = self.instance.RemoveExpectationsFromFile( 603 stale_expectations, self.filename, expectations.RemovalType.STALE) 604 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 605 with open(self.filename) as f: 606 self.assertEqual(f.read(), expected_contents) 607 608 def testLargeGroupBlockAllRemovable(self): 609 """Tests that a large group with all members removable is removed.""" 610 # This test exists because we've had issues that passed tests with 611 # relatively small groups, but failed on larger ones. 612 contents = self.header + """ 613 614# This is a test comment 615crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 616 617# Another comment 618 619# finder:group-start some group name 620[ linux ] a [ RetryOnFailure ] 621[ linux ] b [ RetryOnFailure ] 622[ linux ] c [ RetryOnFailure ] 623[ linux ] d [ RetryOnFailure ] 624[ linux ] e [ RetryOnFailure ] 625[ linux ] f [ RetryOnFailure ] 626[ linux ] g [ RetryOnFailure ] 627[ linux ] h [ RetryOnFailure ] 628[ linux ] i [ RetryOnFailure ] 629[ linux ] j [ RetryOnFailure ] 630[ linux ] k [ RetryOnFailure ] 631# finder:group-end 632[ win ] bar/test [ RetryOnFailure ] 633""" 634 635 stale_expectations = [ 636 data_types.Expectation('a', ['linux'], ['RetryOnFailure']), 637 data_types.Expectation('b', ['linux'], ['RetryOnFailure']), 638 data_types.Expectation('c', ['linux'], ['RetryOnFailure']), 639 data_types.Expectation('d', ['linux'], ['RetryOnFailure']), 640 data_types.Expectation('e', ['linux'], ['RetryOnFailure']), 641 data_types.Expectation('f', ['linux'], ['RetryOnFailure']), 642 data_types.Expectation('g', ['linux'], ['RetryOnFailure']), 643 data_types.Expectation('h', ['linux'], ['RetryOnFailure']), 644 data_types.Expectation('i', ['linux'], ['RetryOnFailure']), 645 data_types.Expectation('j', ['linux'], ['RetryOnFailure']), 646 data_types.Expectation('k', ['linux'], ['RetryOnFailure']), 647 ] 648 649 expected_contents = self.header + """ 650 651# This is a test comment 652crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 653 654# Another comment 655 656[ win ] bar/test [ RetryOnFailure ] 657""" 658 659 with open(self.filename, 'w') as f: 660 f.write(contents) 661 662 removed_urls = self.instance.RemoveExpectationsFromFile( 663 stale_expectations, self.filename, expectations.RemovalType.STALE) 664 self.assertEqual(removed_urls, set([])) 665 with open(self.filename) as f: 666 self.assertEqual(f.read(), expected_contents) 667 668 def testNestedGroupAndNarrowingAllRemovable(self): 669 """Tests that a disable block within a group can be properly removed.""" 670 contents = self.header + """ 671crbug.com/2345 [ win ] baz/test [ Failure ] 672 673# Description 674# finder:group-start name 675# finder:disable-narrowing 676crbug.com/1234 [ win ] foo/test [ Failure ] 677crbug.com/1234 [ win ] bar/test [ Failure ] 678# finder:enable-narrowing 679# finder:group-end 680 681crbug.com/3456 [ linux ] foo/test [ Failure ] 682""" 683 684 stale_expectations = [ 685 data_types.Expectation('foo/test', ['win'], ['Failure'], 686 'crbug.com/1234'), 687 data_types.Expectation('bar/test', ['win'], ['Failure'], 688 'crbug.com/1234'), 689 ] 690 691 expected_contents = self.header + """ 692crbug.com/2345 [ win ] baz/test [ Failure ] 693 694 695crbug.com/3456 [ linux ] foo/test [ Failure ] 696""" 697 698 with open(self.filename, 'w') as f: 699 f.write(contents) 700 701 removed_urls = self.instance.RemoveExpectationsFromFile( 702 stale_expectations, self.filename, expectations.RemovalType.STALE) 703 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 704 with open(self.filename) as f: 705 self.assertEqual(f.read(), expected_contents) 706 707 def testGroupBlockNotAllRemovable(self): 708 """Tests that a group with not all members removable is not removed.""" 709 contents = self.header + """ 710 711# This is a test comment 712crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 713 714# Another comment 715# finder:group-start some group name 716[ linux ] bar/test [ RetryOnFailure ] 717crbug.com/1234 [ win ] foo/test [ Failure ] 718# finder:group-end 719[ win ] bar/test [ RetryOnFailure ] 720""" 721 722 stale_expectations = [ 723 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 724 ] 725 726 expected_contents = contents 727 728 with open(self.filename, 'w') as f: 729 f.write(contents) 730 731 removed_urls = self.instance.RemoveExpectationsFromFile( 732 stale_expectations, self.filename, expectations.RemovalType.STALE) 733 self.assertEqual(removed_urls, set()) 734 with open(self.filename) as f: 735 self.assertEqual(f.read(), expected_contents) 736 737 def testGroupSplitAllRemovable(self): 738 """Tests that a split group with all members removable is removed.""" 739 contents = self.header + """ 740 741# This is a test comment 742crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 743 744# Another comment 745 746# finder:group-start some group name 747[ linux ] bar/test [ RetryOnFailure ] 748# finder:group-end 749 750# finder:group-start some group name 751crbug.com/1234 [ win ] foo/test [ Failure ] 752# finder:group-end 753[ win ] bar/test [ RetryOnFailure ] 754""" 755 756 stale_expectations = [ 757 data_types.Expectation('foo/test', ['win'], ['Failure'], 758 'crbug.com/1234'), 759 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 760 ] 761 762 expected_contents = self.header + """ 763 764# This is a test comment 765crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 766 767# Another comment 768 769 770[ win ] bar/test [ RetryOnFailure ] 771""" 772 773 with open(self.filename, 'w') as f: 774 f.write(contents) 775 776 removed_urls = self.instance.RemoveExpectationsFromFile( 777 stale_expectations, self.filename, expectations.RemovalType.STALE) 778 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 779 with open(self.filename) as f: 780 self.assertEqual(f.read(), expected_contents) 781 782 def testGroupSplitNotAllRemovable(self): 783 """Tests that a split group without all members removable is not removed.""" 784 contents = self.header + """ 785 786# This is a test comment 787crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 788 789# Another comment 790# finder:group-start some group name 791[ linux ] bar/test [ RetryOnFailure ] 792# finder:group-end 793 794# finder:group-start some group name 795crbug.com/1234 [ win ] foo/test [ Failure ] 796# finder:group-end 797[ win ] bar/test [ RetryOnFailure ] 798""" 799 800 stale_expectations = [ 801 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 802 ] 803 804 expected_contents = contents 805 806 with open(self.filename, 'w') as f: 807 f.write(contents) 808 809 removed_urls = self.instance.RemoveExpectationsFromFile( 810 stale_expectations, self.filename, expectations.RemovalType.STALE) 811 self.assertEqual(removed_urls, set()) 812 with open(self.filename) as f: 813 self.assertEqual(f.read(), expected_contents) 814 815 def testGroupMultipleGroupsAllRemovable(self): 816 """Tests that multiple groups with all members removable are removed.""" 817 contents = self.header + """ 818 819# This is a test comment 820crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 821 822# Another comment 823 824# finder:group-start some group name 825[ linux ] bar/test [ RetryOnFailure ] 826# finder:group-end 827 828# finder:group-start another group name 829crbug.com/1234 [ win ] foo/test [ Failure ] 830# finder:group-end 831[ win ] bar/test [ RetryOnFailure ] 832""" 833 834 stale_expectations = [ 835 data_types.Expectation('foo/test', ['win'], ['Failure'], 836 'crbug.com/1234'), 837 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 838 ] 839 840 expected_contents = self.header + """ 841 842# This is a test comment 843crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 844 845# Another comment 846 847 848[ win ] bar/test [ RetryOnFailure ] 849""" 850 851 with open(self.filename, 'w') as f: 852 f.write(contents) 853 854 removed_urls = self.instance.RemoveExpectationsFromFile( 855 stale_expectations, self.filename, expectations.RemovalType.STALE) 856 self.assertEqual(removed_urls, set(['crbug.com/1234'])) 857 with open(self.filename) as f: 858 self.assertEqual(f.read(), expected_contents) 859 860 def testGroupMultipleGroupsSomeRemovable(self): 861 """Tests that multiple groups are handled separately.""" 862 contents = self.header + """ 863 864# This is a test comment 865crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 866 867# Another comment 868 869# finder:group-start some group name 870[ linux ] bar/test [ RetryOnFailure ] 871# finder:group-end 872 873# finder:group-start another group name 874crbug.com/1234 [ win ] foo/test [ Failure ] 875crbug.com/1234 [ linux ] foo/test [ Failure ] 876# finder:group-end 877[ win ] bar/test [ RetryOnFailure ] 878""" 879 880 stale_expectations = [ 881 data_types.Expectation('foo/test', ['win'], ['Failure'], 882 'crbug.com/1234'), 883 data_types.Expectation('bar/test', ['linux'], ['RetryOnFailure']) 884 ] 885 886 expected_contents = self.header + """ 887 888# This is a test comment 889crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 890 891# Another comment 892 893 894# finder:group-start another group name 895crbug.com/1234 [ win ] foo/test [ Failure ] 896crbug.com/1234 [ linux ] foo/test [ Failure ] 897# finder:group-end 898[ win ] bar/test [ RetryOnFailure ] 899""" 900 901 with open(self.filename, 'w') as f: 902 f.write(contents) 903 904 removed_urls = self.instance.RemoveExpectationsFromFile( 905 stale_expectations, self.filename, expectations.RemovalType.STALE) 906 self.assertEqual(removed_urls, set()) 907 with open(self.filename) as f: 908 self.assertEqual(f.read(), expected_contents) 909 910 def testNestedGroupStart(self): 911 """Tests that nested groups are disallowed.""" 912 contents = self.header + """ 913 914# This is a test comment 915crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 916 917# Another comment 918# finder:group-start some group name 919[ linux ] bar/test [ RetryOnFailure ] 920# finder:group-start another group name 921# finder:group-end 922# finder:group-end 923[ win ] bar/test [ RetryOnFailure ] 924""" 925 926 with open(self.filename, 'w') as f: 927 f.write(contents) 928 929 with self.assertRaisesRegex(RuntimeError, 930 'that is inside another group block'): 931 self.instance.RemoveExpectationsFromFile([], self.filename, 932 expectations.RemovalType.STALE) 933 934 def testOrphanedGroupEnd(self): 935 """Tests that orphaned group ends are disallowed.""" 936 contents = self.header + """ 937 938# This is a test comment 939crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 940 941# Another comment 942# finder:group-end 943[ linux ] bar/test [ RetryOnFailure ] 944[ win ] bar/test [ RetryOnFailure ] 945""" 946 947 with open(self.filename, 'w') as f: 948 f.write(contents) 949 950 with self.assertRaisesRegex(RuntimeError, 'without a group start comment'): 951 self.instance.RemoveExpectationsFromFile([], self.filename, 952 expectations.RemovalType.STALE) 953 954 def testNoGroupName(self): 955 """Tests that unnamed groups are disallowed.""" 956 contents = self.header + """ 957 958# This is a test comment 959crbug.com/2345 [ win ] foo/test [ RetryOnFailure ] 960 961# Another comment 962# finder:group-start 963# finder:group-end 964[ linux ] bar/test [ RetryOnFailure ] 965[ win ] bar/test [ RetryOnFailure ] 966""" 967 968 with open(self.filename, 'w') as f: 969 f.write(contents) 970 971 with self.assertRaisesRegex(RuntimeError, 'did not have a group name'): 972 self.instance.RemoveExpectationsFromFile([], self.filename, 973 expectations.RemovalType.STALE) 974 975 def testRemoveCommentBlockSimpleTrailingWhitespace(self): 976 """Tests stale comment removal in a simple case with trailing whitespace.""" 977 contents = self.header + """ 978# Comment line 1 979# Comment line 2 980crbug.com/1234 [ linux ] foo/test [ Failure ] 981 982crbug.com/2345 [ win ] bar/test [ Failure ] 983""" 984 985 stale_expectations = [ 986 data_types.Expectation('foo/test', ['linux'], ['Failure'], 987 'crbug.com/1234'), 988 ] 989 990 expected_contents = self.header + """ 991 992crbug.com/2345 [ win ] bar/test [ Failure ] 993""" 994 995 with open(self.filename, 'w') as f: 996 f.write(contents) 997 998 removed_urls = self.instance.RemoveExpectationsFromFile( 999 stale_expectations, self.filename, expectations.RemovalType.STALE) 1000 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1001 with open(self.filename) as f: 1002 self.assertEqual(f.read(), expected_contents) 1003 1004 def testRemoveCommentBlockSimpleTrailingComment(self): 1005 """Tests stale comment removal in a simple case with trailing comment.""" 1006 contents = self.header + """ 1007# Comment line 1 1008# Comment line 2 1009crbug.com/1234 [ linux ] foo/test [ Failure ] 1010# Comment line 3 1011crbug.com/2345 [ win ] bar/test [ Failure ] 1012""" 1013 1014 stale_expectations = [ 1015 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1016 'crbug.com/1234'), 1017 ] 1018 1019 expected_contents = self.header + """ 1020# Comment line 3 1021crbug.com/2345 [ win ] bar/test [ Failure ] 1022""" 1023 1024 with open(self.filename, 'w') as f: 1025 f.write(contents) 1026 1027 removed_urls = self.instance.RemoveExpectationsFromFile( 1028 stale_expectations, self.filename, expectations.RemovalType.STALE) 1029 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1030 with open(self.filename) as f: 1031 self.assertEqual(f.read(), expected_contents) 1032 1033 def testRemoveCommentBlockSimpleEndOfFile(self): 1034 """Tests stale comment removal in a simple case at file end.""" 1035 contents = self.header + """ 1036crbug.com/2345 [ win ] bar/test [ Failure ] 1037 1038# Comment line 1 1039# Comment line 2 1040crbug.com/1234 [ linux ] foo/test [ Failure ]""" 1041 1042 stale_expectations = [ 1043 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1044 'crbug.com/1234'), 1045 ] 1046 1047 expected_contents = self.header + """ 1048crbug.com/2345 [ win ] bar/test [ Failure ] 1049 1050""" 1051 1052 with open(self.filename, 'w') as f: 1053 f.write(contents) 1054 1055 removed_urls = self.instance.RemoveExpectationsFromFile( 1056 stale_expectations, self.filename, expectations.RemovalType.STALE) 1057 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1058 with open(self.filename) as f: 1059 self.assertEqual(f.read(), expected_contents) 1060 1061 def testRemoveCommentBlockWithAnnotations(self): 1062 """Tests stale comment removal with annotations on both ends.""" 1063 contents = self.header + """ 1064# Comment line 1 1065# Comment line 2 1066# finder:disable-unused 1067crbug.com/1234 [ linux ] foo/test [ Failure ] 1068# finder:enable-unused 1069# Comment line 3 1070crbug.com/2345 [ win ] bar/test [ Failure ] 1071""" 1072 1073 stale_expectations = [ 1074 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1075 'crbug.com/1234'), 1076 ] 1077 1078 expected_contents = self.header + """ 1079# Comment line 3 1080crbug.com/2345 [ win ] bar/test [ Failure ] 1081""" 1082 1083 with open(self.filename, 'w') as f: 1084 f.write(contents) 1085 1086 removed_urls = self.instance.RemoveExpectationsFromFile( 1087 stale_expectations, self.filename, expectations.RemovalType.STALE) 1088 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1089 with open(self.filename) as f: 1090 self.assertEqual(f.read(), expected_contents) 1091 1092 def testRemoveCommentBlockWithMissingTrailingAnnotation(self): 1093 """Tests stale comment removal with a missing trailing annotation.""" 1094 contents = self.header + """ 1095# Comment line 1 1096# Comment line 2 1097# finder:disable-unused 1098crbug.com/1234 [ linux ] foo/test [ Failure ] 1099 1100crbug.com/1234 [ win ] foo/test [ Failure ] 1101# finder:enable-unused 1102 1103# Comment line 3 1104crbug.com/2345 [ win ] bar/test [ Failure ] 1105""" 1106 1107 stale_expectations = [ 1108 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1109 'crbug.com/1234'), 1110 ] 1111 1112 expected_contents = self.header + """ 1113# Comment line 1 1114# Comment line 2 1115# finder:disable-unused 1116 1117crbug.com/1234 [ win ] foo/test [ Failure ] 1118# finder:enable-unused 1119 1120# Comment line 3 1121crbug.com/2345 [ win ] bar/test [ Failure ] 1122""" 1123 1124 with open(self.filename, 'w') as f: 1125 f.write(contents) 1126 1127 removed_urls = self.instance.RemoveExpectationsFromFile( 1128 stale_expectations, self.filename, expectations.RemovalType.STALE) 1129 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1130 with open(self.filename) as f: 1131 self.assertEqual(f.read(), expected_contents) 1132 1133 def testRemoveCommentBlockWithMissingStartAnnotation(self): 1134 """Tests stale comment removal with a missing start annotation.""" 1135 contents = self.header + """ 1136# finder:disable-unused 1137crbug.com/1234 [ win ] foo/test [ Failure ] 1138# Comment line 1 1139# Comment line 2 1140crbug.com/1234 [ linux ] foo/test [ Failure ] 1141# finder:enable-unused 1142# Comment line 3 1143crbug.com/2345 [ win ] bar/test [ Failure ] 1144""" 1145 1146 stale_expectations = [ 1147 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1148 'crbug.com/1234'), 1149 ] 1150 1151 expected_contents = self.header + """ 1152# finder:disable-unused 1153crbug.com/1234 [ win ] foo/test [ Failure ] 1154# finder:enable-unused 1155# Comment line 3 1156crbug.com/2345 [ win ] bar/test [ Failure ] 1157""" 1158 1159 with open(self.filename, 'w') as f: 1160 f.write(contents) 1161 1162 removed_urls = self.instance.RemoveExpectationsFromFile( 1163 stale_expectations, self.filename, expectations.RemovalType.STALE) 1164 self.assertEqual(removed_urls, {'crbug.com/1234'}) 1165 with open(self.filename) as f: 1166 self.assertEqual(f.read(), expected_contents) 1167 1168 def testRemoveCommentBlockMultipleExpectations(self): 1169 """Tests stale comment removal with multiple expectations in a block.""" 1170 contents = self.header + """ 1171# Comment line 1 1172# Comment line 2 1173# finder:disable-unused 1174crbug.com/1234 [ linux ] foo/test [ Failure ] 1175crbug.com/3456 [ mac ] foo/test [ Failure ] 1176# finder:enable-unused 1177 1178# Comment line 3 1179crbug.com/2345 [ win ] bar/test [ Failure ] 1180""" 1181 1182 stale_expectations = [ 1183 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1184 'crbug.com/1234'), 1185 data_types.Expectation('foo/test', ['mac'], ['Failure'], 1186 'crbug.com/3456'), 1187 ] 1188 1189 expected_contents = self.header + """ 1190 1191# Comment line 3 1192crbug.com/2345 [ win ] bar/test [ Failure ] 1193""" 1194 1195 with open(self.filename, 'w') as f: 1196 f.write(contents) 1197 1198 removed_urls = self.instance.RemoveExpectationsFromFile( 1199 stale_expectations, self.filename, expectations.RemovalType.STALE) 1200 self.assertEqual(removed_urls, {'crbug.com/1234', 'crbug.com/3456'}) 1201 with open(self.filename) as f: 1202 self.assertEqual(f.read(), expected_contents) 1203 1204 def testRemoveCommentBlockMultipleBlocks(self): 1205 """Tests stale comment removal with expectations in multiple blocks.""" 1206 contents = self.header + """ 1207# Comment line 1 1208# Comment line 2 1209# finder:disable-unused 1210crbug.com/1234 [ linux ] foo/test [ Failure ] 1211# finder:enable-unused 1212 1213# Comment line 4 1214# finder:disable-unused 1215crbug.com/3456 [ mac ] foo/test [ Failure ] 1216# finder:enable-unused 1217# Comment line 3 1218crbug.com/2345 [ win ] bar/test [ Failure ] 1219""" 1220 1221 stale_expectations = [ 1222 data_types.Expectation('foo/test', ['linux'], ['Failure'], 1223 'crbug.com/1234'), 1224 data_types.Expectation('foo/test', ['mac'], ['Failure'], 1225 'crbug.com/3456'), 1226 ] 1227 1228 expected_contents = self.header + """ 1229 1230# Comment line 3 1231crbug.com/2345 [ win ] bar/test [ Failure ] 1232""" 1233 1234 with open(self.filename, 'w') as f: 1235 f.write(contents) 1236 1237 removed_urls = self.instance.RemoveExpectationsFromFile( 1238 stale_expectations, self.filename, expectations.RemovalType.STALE) 1239 self.assertEqual(removed_urls, {'crbug.com/1234', 'crbug.com/3456'}) 1240 with open(self.filename) as f: 1241 self.assertEqual(f.read(), expected_contents) 1242 1243 def testRemoveStaleAnnotationBlocks(self): 1244 """Tests removal of annotation blocks not associated with removals.""" 1245 contents = self.header + """ 1246# finder:disable-general 1247# finder:enable-general 1248 1249# finder:disable-stale 1250 1251# finder:enable-stale 1252 1253# finder:disable-unused 1254# comment 1255# finder:enable-unused 1256 1257# finder:disable-narrowing description 1258# comment 1259# finder:enable-narrowing 1260 1261# finder:group-start name 1262# finder:group-end 1263""" 1264 1265 stale_expectations = [] 1266 1267 expected_contents = self.header + """ 1268 1269 1270 1271 1272""" 1273 1274 with open(self.filename, 'w') as f: 1275 f.write(contents) 1276 1277 removed_urls = self.instance.RemoveExpectationsFromFile( 1278 stale_expectations, self.filename, expectations.RemovalType.STALE) 1279 self.assertEqual(removed_urls, set()) 1280 with open(self.filename) as f: 1281 self.assertEqual(f.read(), expected_contents) 1282 1283 def testGroupNameExtraction(self): 1284 """Tests that group names are properly extracted.""" 1285 group_name = expectations._GetGroupNameFromCommentLine( 1286 '# finder:group-start group name') 1287 self.assertEqual(group_name, 'group name') 1288 1289 1290class GetDisableAnnotatedExpectationsFromFileUnittest(unittest.TestCase): 1291 def setUp(self) -> None: 1292 self.instance = uu.CreateGenericExpectations() 1293 1294 def testNestedBlockComments(self) -> None: 1295 """Tests that nested disable block comments throw exceptions.""" 1296 contents = """ 1297# finder:disable-general 1298# finder:disable-general 1299crbug.com/1234 [ win ] foo/test [ Failure ] 1300# finder:enable-general 1301# finder:enable-general 1302""" 1303 with self.assertRaises(RuntimeError): 1304 self.instance._GetDisableAnnotatedExpectationsFromFile( 1305 'expectation_file', contents) 1306 1307 contents = """ 1308# finder:disable-general 1309# finder:disable-stale 1310crbug.com/1234 [ win ] foo/test [ Failure ] 1311# finder:enable-stale 1312# finder:enable-general 1313""" 1314 with self.assertRaises(RuntimeError): 1315 self.instance._GetDisableAnnotatedExpectationsFromFile( 1316 'expectation_file', contents) 1317 1318 contents = """ 1319# finder:enable-general 1320crbug.com/1234 [ win ] foo/test [ Failure ] 1321""" 1322 with self.assertRaises(RuntimeError): 1323 self.instance._GetDisableAnnotatedExpectationsFromFile( 1324 'expectation_file', contents) 1325 1326 def testBlockComments(self) -> None: 1327 """Tests that disable block comments are properly parsed.""" 1328 contents = """ 1329# finder:disable-general general-reason 1330crbug.com/1234 [ win ] foo/test [ Failure ] 1331# finder:enable-general 1332 1333# finder:disable-stale 1334crbug.com/1234 [ mac ] foo/test [ Failure ] 1335# finder:enable-stale 1336 1337# finder:disable-unused unused reason 1338crbug.com/1234 [ linux ] foo/test [ Failure ] 1339# finder:enable-unused 1340 1341# finder:disable-narrowing 1342crbug.com/1234 [ win ] bar/test [ Failure ] 1343# finder:enable-narrowing 1344 1345crbug.com/1234 [ mac ] bar/test [ Failure ] 1346""" 1347 annotated_expectations = ( 1348 self.instance._GetDisableAnnotatedExpectationsFromFile( 1349 'expectation_file', contents)) 1350 self.assertEqual(len(annotated_expectations), 4) 1351 self.assertEqual( 1352 annotated_expectations[data_types.Expectation('foo/test', ['win'], 1353 'Failure', 1354 'crbug.com/1234')], 1355 ('-general', 'general-reason')) 1356 self.assertEqual( 1357 annotated_expectations[data_types.Expectation('foo/test', ['mac'], 1358 'Failure', 1359 'crbug.com/1234')], 1360 ('-stale', '')) 1361 self.assertEqual( 1362 annotated_expectations[data_types.Expectation('foo/test', ['linux'], 1363 'Failure', 1364 'crbug.com/1234')], 1365 ('-unused', 'unused reason')) 1366 self.assertEqual( 1367 annotated_expectations[data_types.Expectation('bar/test', ['win'], 1368 'Failure', 1369 'crbug.com/1234')], 1370 ('-narrowing', '')) 1371 1372 def testInlineComments(self) -> None: 1373 """Tests that inline disable comments are properly parsed.""" 1374 # pylint: disable=line-too-long 1375 contents = """ 1376crbug.com/1234 [ win ] foo/test [ Failure ] # finder:disable-general general-reason 1377 1378crbug.com/1234 [ mac ] foo/test [ Failure ] # finder:disable-stale 1379 1380crbug.com/1234 [ linux ] foo/test [ Failure ] # finder:disable-unused unused reason 1381 1382crbug.com/1234 [ win ] bar/test [ Failure ] # finder:disable-narrowing 1383 1384crbug.com/1234 [ mac ] bar/test [ Failure ] 1385""" 1386 # pylint: enable=line-too-long 1387 annotated_expectations = ( 1388 self.instance._GetDisableAnnotatedExpectationsFromFile( 1389 'expectation_file', contents)) 1390 self.assertEqual(len(annotated_expectations), 4) 1391 self.assertEqual( 1392 annotated_expectations[data_types.Expectation('foo/test', ['win'], 1393 'Failure', 1394 'crbug.com/1234')], 1395 ('-general', 'general-reason')) 1396 self.assertEqual( 1397 annotated_expectations[data_types.Expectation('foo/test', ['mac'], 1398 'Failure', 1399 'crbug.com/1234')], 1400 ('-stale', '')) 1401 self.assertEqual( 1402 annotated_expectations[data_types.Expectation('foo/test', ['linux'], 1403 'Failure', 1404 'crbug.com/1234')], 1405 ('-unused', 'unused reason')) 1406 self.assertEqual( 1407 annotated_expectations[data_types.Expectation('bar/test', ['win'], 1408 'Failure', 1409 'crbug.com/1234')], 1410 ('-narrowing', '')) 1411 1412 1413class GetExpectationLineUnittest(unittest.TestCase): 1414 def setUp(self) -> None: 1415 self.instance = uu.CreateGenericExpectations() 1416 1417 def testNoMatchingExpectation(self) -> None: 1418 """Tests that the case of no matching expectation is handled.""" 1419 expectation = data_types.Expectation('foo', ['win'], 'Failure') 1420 line, line_number = self.instance._GetExpectationLine( 1421 expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file') 1422 self.assertIsNone(line) 1423 self.assertIsNone(line_number) 1424 1425 def testMatchingExpectation(self) -> None: 1426 """Tests that matching expectations are found.""" 1427 expectation = data_types.Expectation('foo/test', ['win'], 'Failure', 1428 'crbug.com/1234') 1429 line, line_number = self.instance._GetExpectationLine( 1430 expectation, FAKE_EXPECTATION_FILE_CONTENTS, 'expectation_file') 1431 self.assertEqual(line, 'crbug.com/1234 [ win ] foo/test [ Failure ]') 1432 self.assertEqual(line_number, 3) 1433 1434 1435class FilterToMostSpecificTypTagsUnittest(fake_filesystem_unittest.TestCase): 1436 def setUp(self) -> None: 1437 self._expectations = uu.CreateGenericExpectations() 1438 self.setUpPyfakefs() 1439 with tempfile.NamedTemporaryFile(delete=False, mode='w') as f: 1440 self.filename = f.name 1441 1442 def testBasic(self) -> None: 1443 """Tests that only the most specific tags are kept.""" 1444 expectation_file_contents = """\ 1445# tags: [ win win10 1446# linux 1447# mac ] 1448# tags: [ nvidia nvidia-0x1111 1449# intel intel-0x2222 1450# amd amd-0x3333] 1451# tags: [ release debug ] 1452""" 1453 with open(self.filename, 'w') as outfile: 1454 outfile.write(expectation_file_contents) 1455 tags = frozenset(['win', 'nvidia', 'nvidia-0x1111', 'release']) 1456 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1457 tags, self.filename) 1458 self.assertEqual(filtered_tags, set(['win', 'nvidia-0x1111', 'release'])) 1459 1460 def testSingleTags(self) -> None: 1461 """Tests that functionality works with single tags.""" 1462 expectation_file_contents = """\ 1463# tags: [ tag1_most_specific ] 1464# tags: [ tag2_most_specific ]""" 1465 with open(self.filename, 'w') as outfile: 1466 outfile.write(expectation_file_contents) 1467 1468 tags = frozenset(['tag1_most_specific', 'tag2_most_specific']) 1469 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1470 tags, self.filename) 1471 self.assertEqual(filtered_tags, tags) 1472 1473 def testUnusedTags(self) -> None: 1474 """Tests that functionality works as expected with extra/unused tags.""" 1475 expectation_file_contents = """\ 1476# tags: [ tag1_least_specific tag1_middle_specific tag1_most_specific ] 1477# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ] 1478# tags: [ some_unused_tag ]""" 1479 with open(self.filename, 'w') as outfile: 1480 outfile.write(expectation_file_contents) 1481 1482 tags = frozenset([ 1483 'tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific', 1484 'tag2_least_specific' 1485 ]) 1486 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1487 tags, self.filename) 1488 self.assertEqual(filtered_tags, 1489 set(['tag1_most_specific', 'tag2_middle_specific'])) 1490 1491 def testMissingTags(self) -> None: 1492 """Tests that a file not having all tags is an error.""" 1493 expectation_file_contents = """\ 1494# tags: [ tag1_least_specific tag1_middle_specific ] 1495# tags: [ tag2_least_specific tag2_middle_specific tag2_most_specific ]""" 1496 with open(self.filename, 'w') as outfile: 1497 outfile.write(expectation_file_contents) 1498 1499 tags = frozenset([ 1500 'tag1_least_specific', 'tag1_most_specific', 'tag2_middle_specific', 1501 'tag2_least_specific' 1502 ]) 1503 with self.assertRaisesRegex(RuntimeError, r'.*tag1_most_specific.*'): 1504 self._expectations._FilterToMostSpecificTypTags(tags, self.filename) 1505 1506 def testTagsLowerCased(self) -> None: 1507 """Tests that found tags are lower cased to match internal tags.""" 1508 expectation_file_contents = """\ 1509# tags: [ Win Win10 1510# Linux 1511# Mac ] 1512# tags: [ nvidia nvidia-0x1111 1513# intel intel-0x2222 1514# amd amd-0x3333] 1515# tags: [ release debug ] 1516""" 1517 with open(self.filename, 'w') as outfile: 1518 outfile.write(expectation_file_contents) 1519 tags = frozenset(['win', 'win10', 'nvidia', 'release']) 1520 filtered_tags = self._expectations._FilterToMostSpecificTypTags( 1521 tags, self.filename) 1522 self.assertEqual(filtered_tags, set(['win10', 'nvidia', 'release'])) 1523 1524 1525class NarrowSemiStaleExpectationScopeUnittest(fake_filesystem_unittest.TestCase 1526 ): 1527 def setUp(self) -> None: 1528 self.setUpPyfakefs() 1529 self.instance = uu.CreateGenericExpectations() 1530 1531 with tempfile.NamedTemporaryFile(delete=False, mode='w') as f: 1532 f.write(FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1533 self.filename = f.name 1534 1535 def testEmptyExpectationMap(self) -> None: 1536 """Tests that scope narrowing with an empty map is a no-op.""" 1537 urls = self.instance.NarrowSemiStaleExpectationScope( 1538 data_types.TestExpectationMap({})) 1539 self.assertEqual(urls, set()) 1540 with open(self.filename) as infile: 1541 self.assertEqual(infile.read(), 1542 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1543 1544 def testWildcard(self) -> None: 1545 """Regression test to ensure that wildcards are modified correctly.""" 1546 file_contents = """\ 1547# tags: [ win ] 1548# tags: [ amd intel ] 1549# results: [ Failure ] 1550 1551crbug.com/1234 [ win ] foo/bar* [ Failure ] 1552""" 1553 with open(self.filename, 'w') as f: 1554 f.write(file_contents) 1555 1556 amd_stats = data_types.BuildStats() 1557 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1558 intel_stats = data_types.BuildStats() 1559 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1560 # yapf: disable 1561 test_expectation_map = data_types.TestExpectationMap({ 1562 self.filename: 1563 data_types.ExpectationBuilderMap({ 1564 data_types.Expectation( 1565 'foo/bar*', ['win'], 'Failure', 'crbug.com/1234'): 1566 data_types.BuilderStepMap({ 1567 'win_builder': 1568 data_types.StepBuildStatsMap({ 1569 'amd': amd_stats, 1570 'intel': intel_stats, 1571 }), 1572 }), 1573 }), 1574 }) 1575 # yap: enable 1576 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1577 expected_contents = """\ 1578# tags: [ win ] 1579# tags: [ amd intel ] 1580# results: [ Failure ] 1581 1582crbug.com/1234 [ intel win ] foo/bar* [ Failure ] 1583""" 1584 with open(self.filename) as infile: 1585 self.assertEqual(infile.read(), expected_contents) 1586 self.assertEqual(urls, {'crbug.com/1234'}) 1587 1588 def testMultipleSteps(self) -> None: 1589 """Tests that scope narrowing works across multiple steps.""" 1590 amd_stats = data_types.BuildStats() 1591 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1592 intel_stats = data_types.BuildStats() 1593 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1594 # yapf: disable 1595 test_expectation_map = data_types.TestExpectationMap({ 1596 self.filename: 1597 data_types.ExpectationBuilderMap({ 1598 data_types.Expectation( 1599 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1600 data_types.BuilderStepMap({ 1601 'win_builder': 1602 data_types.StepBuildStatsMap({ 1603 'amd': amd_stats, 1604 'intel': intel_stats, 1605 }), 1606 }), 1607 }), 1608 }) 1609 # yapf: enable 1610 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1611 expected_contents = """\ 1612# tags: [ win win10 1613# linux 1614# mac ] 1615# tags: [ nvidia nvidia-0x1111 1616# intel intel-0x2222 1617# amd amd-0x3333] 1618# tags: [ release debug ] 1619# results: [ Failure RetryOnFailure ] 1620 1621crbug.com/1234 [ intel win ] foo/test [ Failure ] 1622crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1623""" 1624 with open(self.filename) as infile: 1625 self.assertEqual(infile.read(), expected_contents) 1626 self.assertEqual(urls, set(['crbug.com/1234'])) 1627 1628 def testMultipleBuilders(self) -> None: 1629 """Tests that scope narrowing works across multiple builders.""" 1630 amd_stats = data_types.BuildStats() 1631 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1632 intel_stats = data_types.BuildStats() 1633 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1634 # yapf: disable 1635 test_expectation_map = data_types.TestExpectationMap({ 1636 self.filename: 1637 data_types.ExpectationBuilderMap({ 1638 data_types.Expectation( 1639 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1640 data_types.BuilderStepMap({ 1641 'win_amd_builder': 1642 data_types.StepBuildStatsMap({ 1643 'amd': amd_stats, 1644 }), 1645 'win_intel_builder': 1646 data_types.StepBuildStatsMap({ 1647 'intel': intel_stats, 1648 }), 1649 }), 1650 }), 1651 }) 1652 # yapf: enable 1653 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1654 expected_contents = """\ 1655# tags: [ win win10 1656# linux 1657# mac ] 1658# tags: [ nvidia nvidia-0x1111 1659# intel intel-0x2222 1660# amd amd-0x3333] 1661# tags: [ release debug ] 1662# results: [ Failure RetryOnFailure ] 1663 1664crbug.com/1234 [ intel win ] foo/test [ Failure ] 1665crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1666""" 1667 with open(self.filename) as infile: 1668 self.assertEqual(infile.read(), expected_contents) 1669 self.assertEqual(urls, set(['crbug.com/1234'])) 1670 1671 def testMultipleExpectations(self) -> None: 1672 """Tests that scope narrowing works across multiple expectations.""" 1673 amd_stats = data_types.BuildStats() 1674 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1675 failed_amd_stats = data_types.BuildStats() 1676 failed_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd'])) 1677 multi_amd_stats = data_types.BuildStats() 1678 multi_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 1679 multi_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'release'])) 1680 intel_stats = data_types.BuildStats() 1681 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1682 debug_stats = data_types.BuildStats() 1683 debug_stats.AddFailedBuild('1', frozenset(['linux', 'debug'])) 1684 release_stats = data_types.BuildStats() 1685 release_stats.AddPassedBuild(frozenset(['linux', 'release'])) 1686 # yapf: disable 1687 test_expectation_map = data_types.TestExpectationMap({ 1688 self.filename: 1689 data_types.ExpectationBuilderMap({ 1690 data_types.Expectation( 1691 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1692 data_types.BuilderStepMap({ 1693 'win_builder': 1694 data_types.StepBuildStatsMap({ 1695 'amd': amd_stats, 1696 'intel': intel_stats, 1697 }), 1698 }), 1699 # These two expectations are here to ensure that our continue logic 1700 # works as expected when we hit cases we can't handle, i.e. that 1701 # later expectations are still handled properly. 1702 data_types.Expectation('bar/test', ['win'], 'Failure', ''): 1703 data_types.BuilderStepMap({ 1704 'win_builder': 1705 data_types.StepBuildStatsMap({ 1706 'win1': amd_stats, 1707 'win2': failed_amd_stats, 1708 }), 1709 }), 1710 data_types.Expectation('baz/test', ['win'], 'Failure', ''): 1711 data_types.BuilderStepMap({ 1712 'win_builder': 1713 data_types.StepBuildStatsMap({ 1714 'win1': amd_stats, 1715 'win2': multi_amd_stats, 1716 }), 1717 }), 1718 data_types.Expectation( 1719 'foo/test', ['linux'], 'RetryOnFailure', 'crbug.com/2345'): 1720 data_types.BuilderStepMap({ 1721 'linux_builder': 1722 data_types.StepBuildStatsMap({ 1723 'debug': debug_stats, 1724 'release': release_stats, 1725 }), 1726 }), 1727 }), 1728 }) 1729 # yapf: enable 1730 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1731 expected_contents = """\ 1732# tags: [ win win10 1733# linux 1734# mac ] 1735# tags: [ nvidia nvidia-0x1111 1736# intel intel-0x2222 1737# amd amd-0x3333] 1738# tags: [ release debug ] 1739# results: [ Failure RetryOnFailure ] 1740 1741crbug.com/1234 [ intel win ] foo/test [ Failure ] 1742crbug.com/2345 [ debug linux ] foo/test [ RetryOnFailure ] 1743""" 1744 with open(self.filename) as infile: 1745 self.assertEqual(infile.read(), expected_contents) 1746 self.assertEqual(urls, set(['crbug.com/1234', 'crbug.com/2345'])) 1747 1748 def testMultipleOutputLines(self) -> None: 1749 """Tests that scope narrowing works with multiple output lines.""" 1750 amd_stats = data_types.BuildStats() 1751 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1752 intel_stats = data_types.BuildStats() 1753 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1754 nvidia_stats = data_types.BuildStats() 1755 nvidia_stats.AddFailedBuild('1', frozenset(['win', 'nvidia'])) 1756 # yapf: disable 1757 test_expectation_map = data_types.TestExpectationMap({ 1758 self.filename: 1759 data_types.ExpectationBuilderMap({ 1760 data_types.Expectation( 1761 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1762 data_types.BuilderStepMap({ 1763 'win_amd_builder': 1764 data_types.StepBuildStatsMap({ 1765 'amd': amd_stats, 1766 }), 1767 'win_intel_builder': 1768 data_types.StepBuildStatsMap({ 1769 'intel': intel_stats, 1770 }), 1771 'win_nvidia_builder': 1772 data_types.StepBuildStatsMap({ 1773 'nvidia': nvidia_stats, 1774 }) 1775 }), 1776 }), 1777 }) 1778 # yapf: enable 1779 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1780 expected_contents = """\ 1781# tags: [ win win10 1782# linux 1783# mac ] 1784# tags: [ nvidia nvidia-0x1111 1785# intel intel-0x2222 1786# amd amd-0x3333] 1787# tags: [ release debug ] 1788# results: [ Failure RetryOnFailure ] 1789 1790crbug.com/1234 [ intel win ] foo/test [ Failure ] 1791crbug.com/1234 [ nvidia win ] foo/test [ Failure ] 1792crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1793""" 1794 with open(self.filename) as infile: 1795 self.assertEqual(infile.read(), expected_contents) 1796 self.assertEqual(urls, set(['crbug.com/1234'])) 1797 1798 def testMultipleTagSets(self) -> None: 1799 """Tests that multiple tag sets result in a scope narrowing no-op.""" 1800 amd_stats = data_types.BuildStats() 1801 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'release'])) 1802 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'debug'])) 1803 intel_stats = data_types.BuildStats() 1804 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1805 # yapf: disable 1806 test_expectation_map = data_types.TestExpectationMap({ 1807 self.filename: 1808 data_types.ExpectationBuilderMap({ 1809 data_types.Expectation( 1810 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1811 data_types.BuilderStepMap({ 1812 'win_builder': 1813 data_types.StepBuildStatsMap({ 1814 'amd': amd_stats, 1815 'intel': intel_stats, 1816 }), 1817 }), 1818 }), 1819 }) 1820 # yapf: enable 1821 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1822 with open(self.filename) as infile: 1823 self.assertEqual(infile.read(), 1824 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1825 self.assertEqual(urls, set()) 1826 1827 def testAmbiguousTags(self): 1828 """Tests that ambiguous tag sets result in a scope narrowing no-op.""" 1829 amd_stats = data_types.BuildStats() 1830 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1831 bad_amd_stats = data_types.BuildStats() 1832 bad_amd_stats.AddFailedBuild('1', frozenset(['win', 'amd'])) 1833 intel_stats = data_types.BuildStats() 1834 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel'])) 1835 # yapf: disable 1836 test_expectation_map = data_types.TestExpectationMap({ 1837 self.filename: 1838 data_types.ExpectationBuilderMap({ 1839 data_types.Expectation( 1840 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1841 data_types.BuilderStepMap({ 1842 'win_builder': 1843 data_types.StepBuildStatsMap({ 1844 'amd': amd_stats, 1845 'intel': intel_stats, 1846 'bad_amd': bad_amd_stats, 1847 }), 1848 }), 1849 }), 1850 }) 1851 # yapf: enable 1852 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1853 with open(self.filename) as infile: 1854 self.assertEqual(infile.read(), 1855 FAKE_EXPECTATION_FILE_CONTENTS_WITH_COMPLEX_TAGS) 1856 self.assertEqual(urls, set()) 1857 1858 def testRemoveCommonTags(self) -> None: 1859 """Tests that scope narrowing removes common/redundant tags.""" 1860 amd_stats = data_types.BuildStats() 1861 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'desktop'])) 1862 intel_stats = data_types.BuildStats() 1863 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'desktop'])) 1864 # yapf: disable 1865 test_expectation_map = data_types.TestExpectationMap({ 1866 self.filename: 1867 data_types.ExpectationBuilderMap({ 1868 data_types.Expectation( 1869 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1870 data_types.BuilderStepMap({ 1871 'win_builder': 1872 data_types.StepBuildStatsMap({ 1873 'amd': amd_stats, 1874 'intel': intel_stats, 1875 }), 1876 }), 1877 }), 1878 }) 1879 # yapf: enable 1880 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1881 expected_contents = """\ 1882# tags: [ win win10 1883# linux 1884# mac ] 1885# tags: [ nvidia nvidia-0x1111 1886# intel intel-0x2222 1887# amd amd-0x3333] 1888# tags: [ release debug ] 1889# results: [ Failure RetryOnFailure ] 1890 1891crbug.com/1234 [ intel win ] foo/test [ Failure ] 1892crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1893""" 1894 with open(self.filename) as infile: 1895 self.assertEqual(infile.read(), expected_contents) 1896 self.assertEqual(urls, {'crbug.com/1234'}) 1897 1898 def testConsolidateKnownOverlappingTags(self) -> None: 1899 """Tests that scope narrowing consolidates known overlapping tags.""" 1900 1901 # This specific example emulates a dual GPU machine where we remove the 1902 # integrated GPU tag. 1903 def SideEffect(tags): 1904 return tags - {'intel'} 1905 1906 amd_stats = data_types.BuildStats() 1907 amd_stats.AddPassedBuild(frozenset(['win', 'amd'])) 1908 nvidia_dgpu_intel_igpu_stats = data_types.BuildStats() 1909 nvidia_dgpu_intel_igpu_stats.AddFailedBuild( 1910 '1', frozenset(['win', 'nvidia', 'intel'])) 1911 # yapf: disable 1912 test_expectation_map = data_types.TestExpectationMap({ 1913 self.filename: 1914 data_types.ExpectationBuilderMap({ 1915 data_types.Expectation( 1916 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1917 data_types.BuilderStepMap({ 1918 'win_builder': 1919 data_types.StepBuildStatsMap({ 1920 'amd': amd_stats, 1921 'dual_gpu': nvidia_dgpu_intel_igpu_stats, 1922 }), 1923 }), 1924 }), 1925 }) 1926 # yapf: enable 1927 with mock.patch.object(self.instance, 1928 '_ConsolidateKnownOverlappingTags', 1929 side_effect=SideEffect): 1930 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1931 expected_contents = """\ 1932# tags: [ win win10 1933# linux 1934# mac ] 1935# tags: [ nvidia nvidia-0x1111 1936# intel intel-0x2222 1937# amd amd-0x3333] 1938# tags: [ release debug ] 1939# results: [ Failure RetryOnFailure ] 1940 1941crbug.com/1234 [ nvidia win ] foo/test [ Failure ] 1942crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1943""" 1944 with open(self.filename) as infile: 1945 self.assertEqual(infile.read(), expected_contents) 1946 self.assertEqual(urls, set(['crbug.com/1234'])) 1947 1948 def testFilterToSpecificTags(self) -> None: 1949 """Tests that scope narrowing filters to the most specific tags.""" 1950 amd_stats = data_types.BuildStats() 1951 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'amd-0x1111'])) 1952 intel_stats = data_types.BuildStats() 1953 intel_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'intel-0x2222'])) 1954 # yapf: disable 1955 test_expectation_map = data_types.TestExpectationMap({ 1956 self.filename: 1957 data_types.ExpectationBuilderMap({ 1958 data_types.Expectation( 1959 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 1960 data_types.BuilderStepMap({ 1961 'win_builder': 1962 data_types.StepBuildStatsMap({ 1963 'amd': amd_stats, 1964 'intel': intel_stats, 1965 }), 1966 }), 1967 }), 1968 }) 1969 # yapf: enable 1970 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 1971 expected_contents = """\ 1972# tags: [ win win10 1973# linux 1974# mac ] 1975# tags: [ nvidia nvidia-0x1111 1976# intel intel-0x2222 1977# amd amd-0x3333] 1978# tags: [ release debug ] 1979# results: [ Failure RetryOnFailure ] 1980 1981crbug.com/1234 [ intel-0x2222 win ] foo/test [ Failure ] 1982crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 1983""" 1984 with open(self.filename) as infile: 1985 self.assertEqual(infile.read(), expected_contents) 1986 self.assertEqual(urls, set(['crbug.com/1234'])) 1987 1988 def testSupersetsRemoved(self) -> None: 1989 """Tests that superset tags (i.e. conflicts) are omitted.""" 1990 # These stats are set up so that the raw new tag sets are: 1991 # [{win, amd}, {win, amd, debug}] since the passed Intel build also has 1992 # "release". Thus, if we aren't correctly filtering out supersets, we'll 1993 # end up with [ amd win ] and [ amd debug win ] in the expectation file 1994 # instead of just [ amd win ]. 1995 amd_release_stats = data_types.BuildStats() 1996 amd_release_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'release'])) 1997 amd_debug_stats = data_types.BuildStats() 1998 amd_debug_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 1999 intel_stats = data_types.BuildStats() 2000 intel_stats.AddPassedBuild(frozenset(['win', 'intel', 'release'])) 2001 # yapf: disable 2002 test_expectation_map = data_types.TestExpectationMap({ 2003 self.filename: 2004 data_types.ExpectationBuilderMap({ 2005 data_types.Expectation( 2006 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2007 data_types.BuilderStepMap({ 2008 'win_builder': 2009 data_types.StepBuildStatsMap({ 2010 'amd_release': amd_release_stats, 2011 'amd_debug': amd_debug_stats, 2012 'intel': intel_stats, 2013 }), 2014 }), 2015 }), 2016 }) 2017 # yapf: enable 2018 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2019 expected_contents = """\ 2020# tags: [ win win10 2021# linux 2022# mac ] 2023# tags: [ nvidia nvidia-0x1111 2024# intel intel-0x2222 2025# amd amd-0x3333] 2026# tags: [ release debug ] 2027# results: [ Failure RetryOnFailure ] 2028 2029crbug.com/1234 [ amd win ] foo/test [ Failure ] 2030crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2031""" 2032 with open(self.filename) as infile: 2033 self.assertEqual(infile.read(), expected_contents) 2034 self.assertEqual(urls, set(['crbug.com/1234'])) 2035 2036 def testNoPassingOverlap(self): 2037 """Tests that scope narrowing works with no overlap between passing tags.""" 2038 # There is no commonality between [ amd debug ] and [ intel release ], so 2039 # the resulting expectation we generate should just be the tags from the 2040 # failed build. 2041 amd_stats = data_types.BuildStats() 2042 amd_stats.AddPassedBuild(frozenset(['win', 'amd', 'debug'])) 2043 intel_stats_debug = data_types.BuildStats() 2044 intel_stats_debug.AddFailedBuild('1', frozenset(['win', 'intel', 'debug'])) 2045 intel_stats_release = data_types.BuildStats() 2046 intel_stats_release.AddPassedBuild(frozenset(['win', 'intel', 'release'])) 2047 # yapf: disable 2048 test_expectation_map = data_types.TestExpectationMap({ 2049 self.filename: 2050 data_types.ExpectationBuilderMap({ 2051 data_types.Expectation( 2052 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2053 data_types.BuilderStepMap({ 2054 'win_builder': 2055 data_types.StepBuildStatsMap({ 2056 'amd': amd_stats, 2057 'intel_debug': intel_stats_debug, 2058 'intel_release': intel_stats_release, 2059 }), 2060 }), 2061 }), 2062 }) 2063 # yapf: enable 2064 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2065 expected_contents = """\ 2066# tags: [ win win10 2067# linux 2068# mac ] 2069# tags: [ nvidia nvidia-0x1111 2070# intel intel-0x2222 2071# amd amd-0x3333] 2072# tags: [ release debug ] 2073# results: [ Failure RetryOnFailure ] 2074 2075crbug.com/1234 [ debug intel win ] foo/test [ Failure ] 2076crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2077""" 2078 with open(self.filename) as infile: 2079 self.assertEqual(infile.read(), expected_contents) 2080 self.assertEqual(urls, set(['crbug.com/1234'])) 2081 2082 def testMultipleOverlap(self): 2083 """Tests that scope narrowing works with multiple potential overlaps.""" 2084 # [ win intel debug ], [ win intel release ], and [ win amd debug ] each 2085 # have 2/3 tags overlapping with each other, so we expect one pair to be 2086 # simplified and the other to remain the same. 2087 intel_debug_stats = data_types.BuildStats() 2088 intel_debug_stats.AddFailedBuild('1', frozenset(['win', 'intel', 'debug'])) 2089 intel_release_stats = data_types.BuildStats() 2090 intel_release_stats.AddFailedBuild('1', 2091 frozenset(['win', 'intel', 'release'])) 2092 amd_debug_stats = data_types.BuildStats() 2093 amd_debug_stats.AddFailedBuild('1', frozenset(['win', 'amd', 'debug'])) 2094 amd_release_stats = data_types.BuildStats() 2095 amd_release_stats.AddPassedBuild(frozenset(['win', 'amd', 'release'])) 2096 # yapf: disable 2097 test_expectation_map = data_types.TestExpectationMap({ 2098 self.filename: 2099 data_types.ExpectationBuilderMap({ 2100 data_types.Expectation( 2101 'foo/test', ['win'], 'Failure', 'crbug.com/1234'): 2102 data_types.BuilderStepMap({ 2103 'win_builder': 2104 data_types.StepBuildStatsMap({ 2105 'amd_debug': amd_debug_stats, 2106 'amd_release': amd_release_stats, 2107 'intel_debug': intel_debug_stats, 2108 'intel_release': intel_release_stats, 2109 }), 2110 }), 2111 }), 2112 }) 2113 # yapf: enable 2114 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2115 # Python sets are not stable between different processes due to random hash 2116 # seeds that are on by default. Since there are two valid ways to simplify 2117 # the tags we provided, this means that the test is flaky if we only check 2118 # for one due to the non-deterministic order the tags are processed, so 2119 # instead, accept either valid output. 2120 # 2121 # Random hash seeds can be disabled by setting PYTHONHASHSEED, but that 2122 # requires that we either ensure that this test is always run with that set 2123 # (difficult/error-prone), or we manually set the seed and recreate the 2124 # process (hacky). Simply accepting either valid value instead of trying to 2125 # force a certain order seems like the better approach. 2126 expected_contents1 = """\ 2127# tags: [ win win10 2128# linux 2129# mac ] 2130# tags: [ nvidia nvidia-0x1111 2131# intel intel-0x2222 2132# amd amd-0x3333] 2133# tags: [ release debug ] 2134# results: [ Failure RetryOnFailure ] 2135 2136crbug.com/1234 [ amd debug win ] foo/test [ Failure ] 2137crbug.com/1234 [ intel win ] foo/test [ Failure ] 2138crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2139""" 2140 expected_contents2 = """\ 2141# tags: [ win win10 2142# linux 2143# mac ] 2144# tags: [ nvidia nvidia-0x1111 2145# intel intel-0x2222 2146# amd amd-0x3333] 2147# tags: [ release debug ] 2148# results: [ Failure RetryOnFailure ] 2149 2150crbug.com/1234 [ debug win ] foo/test [ Failure ] 2151crbug.com/1234 [ intel release win ] foo/test [ Failure ] 2152crbug.com/2345 [ linux ] foo/test [ RetryOnFailure ] 2153""" 2154 with open(self.filename) as infile: 2155 self.assertIn(infile.read(), (expected_contents1, expected_contents2)) 2156 self.assertEqual(urls, set(['crbug.com/1234'])) 2157 2158 def testMultipleOverlapRepeatedIntersection(self): 2159 """Edge case where intersection checks need to be repeated to work.""" 2160 original_contents = """\ 2161# tags: [ mac 2162# win ] 2163# tags: [ amd amd-0x3333 2164# intel intel-0x2222 intel-0x4444 2165# nvidia nvidia-0x1111 ] 2166# results: [ Failure ] 2167 2168crbug.com/1234 foo/test [ Failure ] 2169""" 2170 with open(self.filename, 'w') as outfile: 2171 outfile.write(original_contents) 2172 amd_stats = data_types.BuildStats() 2173 amd_stats.AddFailedBuild('1', frozenset(['mac', 'amd', 'amd-0x3333'])) 2174 intel_stats_1 = data_types.BuildStats() 2175 intel_stats_1.AddFailedBuild('1', 2176 frozenset(['mac', 'intel', 'intel-0x2222'])) 2177 intel_stats_2 = data_types.BuildStats() 2178 intel_stats_2.AddFailedBuild('1', 2179 frozenset(['mac', 'intel', 'intel-0x4444'])) 2180 nvidia_stats = data_types.BuildStats() 2181 nvidia_stats.AddPassedBuild(frozenset(['win', 'nvidia', 'nvidia-0x1111'])) 2182 2183 # yapf: disable 2184 test_expectation_map = data_types.TestExpectationMap({ 2185 self.filename: 2186 data_types.ExpectationBuilderMap({ 2187 data_types.Expectation( 2188 'foo/test', [], 'Failure', 'crbug.com/1234'): 2189 data_types.BuilderStepMap({ 2190 'mixed_builder': 2191 data_types.StepBuildStatsMap({ 2192 'amd': amd_stats, 2193 'intel_1': intel_stats_1, 2194 'intel_2': intel_stats_2, 2195 'nvidia': nvidia_stats, 2196 }), 2197 }), 2198 }), 2199 }) 2200 # yapf: enable 2201 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2202 expected_contents = """\ 2203# tags: [ mac 2204# win ] 2205# tags: [ amd amd-0x3333 2206# intel intel-0x2222 intel-0x4444 2207# nvidia nvidia-0x1111 ] 2208# results: [ Failure ] 2209 2210crbug.com/1234 [ mac ] foo/test [ Failure ] 2211""" 2212 with open(self.filename) as infile: 2213 self.assertEqual(infile.read(), expected_contents) 2214 self.assertEqual(urls, set(['crbug.com/1234'])) 2215 2216 def testBlockDisableAnnotation(self) -> None: 2217 """Tests that narrowing is skipped if block annotations are present.""" 2218 original_contents = """\ 2219# tags: [ mac ] 2220# tags: [ amd intel ] 2221# results: [ Failure ] 2222 2223crbug.com/1234 [ mac ] foo/test [ Failure ] 2224# finder:disable-narrowing 2225crbug.com/2345 [ mac ] bar/test [ Failure ] 2226# finder:enable-narrowing 2227""" 2228 with open(self.filename, 'w') as outfile: 2229 outfile.write(original_contents) 2230 2231 amd_stats = data_types.BuildStats() 2232 amd_stats.AddPassedBuild(frozenset(['mac', 'amd'])) 2233 intel_stats = data_types.BuildStats() 2234 intel_stats.AddFailedBuild('1', frozenset(['mac', 'intel'])) 2235 2236 # yapf: disable 2237 test_expectation_map = data_types.TestExpectationMap({ 2238 self.filename: 2239 data_types.ExpectationBuilderMap({ 2240 data_types.Expectation( 2241 'foo/test', ['mac'], 'Failure', 'crbug.com/1234'): 2242 data_types.BuilderStepMap({ 2243 'mac_builder': 2244 data_types.StepBuildStatsMap({ 2245 'amd': amd_stats, 2246 'intel': intel_stats, 2247 }), 2248 }), 2249 data_types.Expectation( 2250 'bar/test', ['mac'], 'Failure', 'crbug.com/2345'): 2251 data_types.BuilderStepMap({ 2252 'mac_builder': 2253 data_types.StepBuildStatsMap({ 2254 'amd': amd_stats, 2255 'intel': intel_stats, 2256 }), 2257 }), 2258 }), 2259 }) 2260 # yapf: enable 2261 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2262 expected_contents = """\ 2263# tags: [ mac ] 2264# tags: [ amd intel ] 2265# results: [ Failure ] 2266 2267crbug.com/1234 [ intel mac ] foo/test [ Failure ] 2268# finder:disable-narrowing 2269crbug.com/2345 [ mac ] bar/test [ Failure ] 2270# finder:enable-narrowing 2271""" 2272 with open(self.filename) as infile: 2273 self.assertEqual(infile.read(), expected_contents) 2274 self.assertEqual(urls, set(['crbug.com/1234'])) 2275 2276 def testNoOverlapsInNarrowedExpectations(self): 2277 """Tests that scope narrowing does not produce overlapping tag sets.""" 2278 original_contents = """\ 2279# tags: [ Linux 2280# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2281# Win Win10.20h2 Win11 ] 2282# tags: [ Release Debug ] 2283# results: [ Failure ] 2284 2285crbug.com/874695 foo/test [ Failure ] 2286""" 2287 with open(self.filename, 'w') as outfile: 2288 outfile.write(original_contents) 2289 2290 linux_debug_stats = data_types.BuildStats() 2291 linux_debug_stats.AddPassedBuild(frozenset(['debug', 'linux'])) 2292 linux_release_stats = data_types.BuildStats() 2293 linux_release_stats.AddFailedBuild('1', frozenset(['linux', 'release'])) 2294 mac10_release_stats = data_types.BuildStats() 2295 mac10_release_stats.AddFailedBuild( 2296 '1', frozenset(['mac', 'mac10.15', 'release'])) 2297 mac11_arm_release_stats = data_types.BuildStats() 2298 mac11_arm_release_stats.AddFailedBuild( 2299 '1', frozenset(['mac', 'mac11-arm64', 'release'])) 2300 mac11_release_stats = data_types.BuildStats() 2301 mac11_release_stats.AddFailedBuild('1', 2302 frozenset(['mac', 'mac11', 'release'])) 2303 mac12_arm_release_stats = data_types.BuildStats() 2304 mac12_arm_release_stats.AddFailedBuild( 2305 '1', frozenset(['mac', 'mac12-arm64', 'release'])) 2306 mac12_debug_stats = data_types.BuildStats() 2307 mac12_debug_stats.AddFailedBuild('1', frozenset(['debug', 'mac', 'mac12'])) 2308 mac12_release_stats = data_types.BuildStats() 2309 mac12_release_stats.AddFailedBuild('1', 2310 frozenset(['mac', 'mac12', 'release'])) 2311 win10_release_stats = data_types.BuildStats() 2312 win10_release_stats.AddFailedBuild( 2313 '1', frozenset(['release', 'win', 'win10.20h2'])) 2314 win11_release_stats = data_types.BuildStats() 2315 win11_release_stats.AddFailedBuild('1', 2316 frozenset(['release', 'win', 'win11'])) 2317 # yapf: disable 2318 test_expectation_map = data_types.TestExpectationMap({ 2319 self.filename: 2320 data_types.ExpectationBuilderMap({ 2321 data_types.Expectation( 2322 'foo/test', 2323 [], 'Failure', 'crbug.com/874695'): 2324 data_types.BuilderStepMap({ 2325 'Linux Tests (dbg)(1)': 2326 data_types.StepBuildStatsMap({ 2327 'blink_web_tests': linux_debug_stats, 2328 }), 2329 'Mac10.15 Tests': 2330 data_types.StepBuildStatsMap({ 2331 'blink_web_tests': mac10_release_stats, 2332 }), 2333 'mac11-arm64-rel-tests': 2334 data_types.StepBuildStatsMap({ 2335 'blink_web_tests': mac11_arm_release_stats, 2336 }), 2337 'Mac11 Tests': 2338 data_types.StepBuildStatsMap({ 2339 'blink_web_tests': mac11_release_stats, 2340 }), 2341 'mac12-arm64-rel-tests': 2342 data_types.StepBuildStatsMap({ 2343 'blink_web_tests': mac12_arm_release_stats, 2344 }), 2345 'Mac12 Tests (dbg)': 2346 data_types.StepBuildStatsMap({ 2347 'blink_web_tests': mac12_debug_stats, 2348 }), 2349 'Mac12 Tests': 2350 data_types.StepBuildStatsMap({ 2351 'blink_web_tests': mac12_release_stats, 2352 }), 2353 'Linux Tests': 2354 data_types.StepBuildStatsMap({ 2355 'blink_web_tests': linux_release_stats, 2356 }), 2357 'WebKit Win10': 2358 data_types.StepBuildStatsMap({ 2359 'blink_web_tests': win10_release_stats, 2360 }), 2361 'Win11 Tests x64': 2362 data_types.StepBuildStatsMap({ 2363 'blink_web_tests': win11_release_stats, 2364 }), 2365 }), 2366 }), 2367 }) 2368 # yapf: enable 2369 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2370 # Python sets are not stable between different processes due to random hash 2371 # seeds that are on by default. Since there are two valid ways to simplify 2372 # the tags we provided, this means that the test is flaky if we only check 2373 # for one due to the non-deterministic order the tags are processed, so 2374 # instead, accept either valid output. 2375 # 2376 # Random hash seeds can be disabled by setting PYTHONHASHSEED, but that 2377 # requires that we either ensure that this test is always run with that set 2378 # (difficult/error-prone), or we manually set the seed and recreate the 2379 # process (hacky). Simply accepting either valid value instead of trying to 2380 # force a certain order seems like the better approach. 2381 expected_contents1 = """\ 2382# tags: [ Linux 2383# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2384# Win Win10.20h2 Win11 ] 2385# tags: [ Release Debug ] 2386# results: [ Failure ] 2387 2388crbug.com/874695 [ debug mac12 ] foo/test [ Failure ] 2389crbug.com/874695 [ release ] foo/test [ Failure ] 2390""" 2391 expected_contents2 = """\ 2392# tags: [ Linux 2393# Mac Mac10.15 Mac11 Mac11-arm64 Mac12 Mac12-arm64 2394# Win Win10.20h2 Win11 ] 2395# tags: [ Release Debug ] 2396# results: [ Failure ] 2397 2398crbug.com/874695 [ linux release ] foo/test [ Failure ] 2399crbug.com/874695 [ mac ] foo/test [ Failure ] 2400crbug.com/874695 [ release win ] foo/test [ Failure ] 2401""" 2402 with open(self.filename) as infile: 2403 self.assertIn(infile.read(), (expected_contents1, expected_contents2)) 2404 self.assertEqual(urls, set(['crbug.com/874695'])) 2405 2406 def testInlineDisableAnnotation(self) -> None: 2407 """Tests that narrowing is skipped if inline annotations are present.""" 2408 original_contents = """\ 2409# tags: [ mac ] 2410# tags: [ amd intel ] 2411# results: [ Failure ] 2412 2413crbug.com/1234 [ mac ] foo/test [ Failure ] 2414crbug.com/2345 [ mac ] bar/test [ Failure ] # finder:disable-narrowing 2415""" 2416 with open(self.filename, 'w') as outfile: 2417 outfile.write(original_contents) 2418 2419 amd_stats = data_types.BuildStats() 2420 amd_stats.AddPassedBuild(frozenset(['mac', 'amd'])) 2421 intel_stats = data_types.BuildStats() 2422 intel_stats.AddFailedBuild('1', frozenset(['mac', 'intel'])) 2423 2424 # yapf: disable 2425 test_expectation_map = data_types.TestExpectationMap({ 2426 self.filename: 2427 data_types.ExpectationBuilderMap({ 2428 data_types.Expectation( 2429 'foo/test', ['mac'], 'Failure', 'crbug.com/1234'): 2430 data_types.BuilderStepMap({ 2431 'mac_builder': 2432 data_types.StepBuildStatsMap({ 2433 'amd': amd_stats, 2434 'intel': intel_stats, 2435 }), 2436 }), 2437 data_types.Expectation( 2438 'bar/test', ['mac'], 'Failure', 'crbug.com/2345'): 2439 data_types.BuilderStepMap({ 2440 'mac_builder': 2441 data_types.StepBuildStatsMap({ 2442 'amd': amd_stats, 2443 'intel': intel_stats, 2444 }), 2445 }), 2446 }), 2447 }) 2448 # yapf: enable 2449 urls = self.instance.NarrowSemiStaleExpectationScope(test_expectation_map) 2450 expected_contents = """\ 2451# tags: [ mac ] 2452# tags: [ amd intel ] 2453# results: [ Failure ] 2454 2455crbug.com/1234 [ intel mac ] foo/test [ Failure ] 2456crbug.com/2345 [ mac ] bar/test [ Failure ] # finder:disable-narrowing 2457""" 2458 with open(self.filename) as infile: 2459 self.assertEqual(infile.read(), expected_contents) 2460 self.assertEqual(urls, set(['crbug.com/1234'])) 2461 2462 2463class FindOrphanedBugsUnittest(fake_filesystem_unittest.TestCase): 2464 def CreateFile(self, *args, **kwargs) -> None: 2465 # TODO(crbug.com/1156806): Remove this and just use fs.create_file() when 2466 # Catapult is updated to a newer version of pyfakefs that is compatible with 2467 # Chromium's version. 2468 if hasattr(self.fs, 'create_file'): 2469 self.fs.create_file(*args, **kwargs) 2470 else: 2471 self.fs.CreateFile(*args, **kwargs) 2472 2473 def setUp(self) -> None: 2474 expectations_dir = os.path.join(os.path.dirname(__file__), 'expectations') 2475 self.setUpPyfakefs() 2476 self.instance = expectations.Expectations() 2477 self.filepath_patcher = mock.patch.object( 2478 self.instance, 2479 'GetExpectationFilepaths', 2480 return_value=[os.path.join(expectations_dir, 'real_expectations.txt')]) 2481 self.filepath_mock = self.filepath_patcher.start() 2482 self.addCleanup(self.filepath_patcher.stop) 2483 2484 real_contents = 'crbug.com/1\ncrbug.com/2' 2485 skipped_contents = 'crbug.com/4' 2486 self.CreateFile(os.path.join(expectations_dir, 'real_expectations.txt'), 2487 contents=real_contents) 2488 self.CreateFile(os.path.join(expectations_dir, 'fake.txt'), 2489 contents=skipped_contents) 2490 2491 def testNoOrphanedBugs(self) -> None: 2492 bugs = ['crbug.com/1', 'crbug.com/2'] 2493 self.assertEqual(self.instance.FindOrphanedBugs(bugs), set()) 2494 2495 def testOrphanedBugs(self) -> None: 2496 bugs = ['crbug.com/1', 'crbug.com/3', 'crbug.com/4'] 2497 self.assertEqual(self.instance.FindOrphanedBugs(bugs), 2498 set(['crbug.com/3', 'crbug.com/4'])) 2499 2500 2501if __name__ == '__main__': 2502 unittest.main(verbosity=2) 2503