1#!/usr/bin/env vpython3 2 3# Copyright 2021 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import os 9from pyfakefs import fake_filesystem_unittest 10import tempfile 11import unittest 12 13import test_filtering 14from test_filtering import _TestFilter 15from test_filtering import _TestFiltersGroup 16from test_filtering import _SetOfTestFiltersGroups 17 18 19class FilterTests(fake_filesystem_unittest.TestCase): 20 def test_exact_match(self): 21 t = _TestFilter('foo') 22 self.assertTrue(t.is_match('foo')) 23 self.assertFalse(t.is_match('foobar')) 24 self.assertFalse(t.is_match('foo/bar')) 25 self.assertFalse(t.is_match('fo')) 26 self.assertFalse(t.is_exclusion_filter()) 27 28 def test_prefix_match(self): 29 t = _TestFilter('foo*') 30 self.assertTrue(t.is_match('foo')) 31 self.assertTrue(t.is_match('foobar')) 32 self.assertTrue(t.is_match('foo/bar')) 33 self.assertFalse(t.is_match('fo')) 34 self.assertFalse(t.is_exclusion_filter()) 35 36 def test_exclusion_match(self): 37 t = _TestFilter('-foo') 38 self.assertTrue(t.is_match('foo')) 39 self.assertFalse(t.is_match('foobar')) 40 self.assertFalse(t.is_match('foo/bar')) 41 self.assertFalse(t.is_match('fo')) 42 self.assertTrue(t.is_exclusion_filter()) 43 44 def test_exclusion_prefix_match(self): 45 t = _TestFilter('-foo*') 46 self.assertTrue(t.is_match('foo')) 47 self.assertTrue(t.is_match('foobar')) 48 self.assertTrue(t.is_match('foo/bar')) 49 self.assertFalse(t.is_match('fo')) 50 self.assertTrue(t.is_exclusion_filter()) 51 52 def test_error_conditions(self): 53 # '*' is only supported at the end 54 with self.assertRaises(ValueError): 55 _TestFilter('*.bar') 56 57 58def _create_group_from_pseudo_file(file_contents): 59 # pylint: disable=unexpected-keyword-arg 60 with tempfile.NamedTemporaryFile(delete=False, mode='w', 61 encoding='utf-8') as f: 62 filepath = f.name 63 f.write(file_contents) 64 try: 65 return _TestFiltersGroup.from_filter_file(filepath) 66 finally: 67 os.remove(filepath) 68 69 70class FiltersGroupTests(fake_filesystem_unittest.TestCase): 71 def test_single_positive_filter(self): 72 g = _TestFiltersGroup.from_string('foo*') 73 self.assertTrue(g.is_test_name_included('foo')) 74 self.assertTrue(g.is_test_name_included('foobar')) 75 self.assertFalse(g.is_test_name_included('baz')) 76 self.assertFalse(g.is_test_name_included('fo')) 77 78 def test_single_negative_filter(self): 79 g = _TestFiltersGroup.from_string('-foo*') 80 self.assertFalse(g.is_test_name_included('foo')) 81 self.assertFalse(g.is_test_name_included('foobar')) 82 self.assertTrue(g.is_test_name_included('baz')) 83 self.assertTrue(g.is_test_name_included('fo')) 84 85 def test_specificity_ordering(self): 86 # From test_executable_api.md#filtering-which-tests-to-run: 87 # 88 # If multiple filters in a flag match a given test name, the longest 89 # match takes priority (longest match wins). I.e.,. if you had 90 # --isolated-script-test-filter='a*::-ab*', then ace.html would run 91 # but abd.html would not. The order of the filters should not 92 # matter. 93 g1 = _TestFiltersGroup.from_string('a*::-ab*') # order #1 94 g2 = _TestFiltersGroup.from_string('-ab*::a*') # order #2 95 self.assertTrue(g1.is_test_name_included('ace')) 96 self.assertTrue(g2.is_test_name_included('ace')) 97 self.assertFalse(g1.is_test_name_included('abd')) 98 self.assertFalse(g2.is_test_name_included('abd')) 99 100 def test_specificity_conflicts(self): 101 # Docs give this specific example: It is an error to have multiple 102 # expressions of the same length that conflict (e.g., a*::-a*). 103 with self.assertRaises(ValueError): 104 _TestFiltersGroup.from_string('a*::-a*') 105 # Other similar conflict: 106 with self.assertRaises(ValueError): 107 _TestFiltersGroup.from_string('a::-a') 108 # It doesn't really make sense to support `foo.bar` and `foo.bar*` and 109 # have the latter take precedence over the former. 110 with self.assertRaises(ValueError): 111 _TestFiltersGroup.from_string('foo.bar::foo.bar*') 112 # In the same spirit, identical duplicates are also treated as 113 # conflicts. 114 with self.assertRaises(ValueError): 115 _TestFiltersGroup.from_string('foo.bar::foo.bar') 116 117 # Ok - no conflicts: 118 _TestFiltersGroup.from_string('a*::-b*') # Different filter text 119 120 def test_simple_list(self): 121 # 'simple test list' format from bit.ly/chromium-test-list-format 122 # (aka go/test-list-format) 123 file_content = """ 124# Comment 125 126aaa 127bbb # End-of-line comment 128Bug(intentional) ccc [ Crash ] # Comment 129crbug.com/12345 [ Debug] ddd 130skbug.com/12345 [ Debug] eee 131webkit.org/12345 [ Debug] fff 132ggg* 133-ggghhh 134""".strip() 135 g = _create_group_from_pseudo_file(file_content) 136 self.assertTrue(g.is_test_name_included('aaa')) 137 self.assertTrue(g.is_test_name_included('bbb')) 138 self.assertTrue(g.is_test_name_included('ccc')) 139 self.assertTrue(g.is_test_name_included('ddd')) 140 141 self.assertFalse(g.is_test_name_included('aa')) 142 self.assertFalse(g.is_test_name_included('aaax')) 143 144 self.assertTrue(g.is_test_name_included('ggg')) 145 self.assertTrue(g.is_test_name_included('gggg')) 146 self.assertFalse(g.is_test_name_included('gg')) 147 self.assertFalse(g.is_test_name_included('ggghhh')) 148 149 self.assertFalse(g.is_test_name_included('zzz')) 150 151 def test_tagged_list(self): 152 # tagged list format from bit.ly/chromium-test-list-format 153 # (aka go/test-list-format) 154 file_content = """ 155# Comment 156 157abc 158foo* # End-of-line comment 159-foobar* 160""".strip() 161 g = _create_group_from_pseudo_file(file_content) 162 self.assertTrue(g.is_test_name_included('abc')) 163 self.assertFalse(g.is_test_name_included('abcd')) 164 self.assertTrue(g.is_test_name_included('foo')) 165 self.assertTrue(g.is_test_name_included('food')) 166 self.assertFalse(g.is_test_name_included('foobar')) 167 self.assertFalse(g.is_test_name_included('foobarbaz')) 168 169 170class SetOfFilterGroupsTests(fake_filesystem_unittest.TestCase): 171 @classmethod 172 def setUpClass(cls): 173 # `_filter1`, `_filter2`, and `_tests` are based on the setup described 174 # in test_executable_api.md#examples 175 cls._filter1 = _TestFiltersGroup.from_string( 176 'Foo.Bar.*::-Foo.Bar.bar3') 177 cls._filter2 = _TestFiltersGroup.from_string('Foo.Bar.bar2') 178 cls._tests = [ 179 'Foo.Bar.bar1', 180 'Foo.Bar.bar2', 181 'Foo.Bar.bar3', 182 'Foo.Baz.baz', # TODO: Fix typo in test_executable_api.md 183 'Foo.Quux.quux' 184 ] 185 186 def test_basics(self): 187 # This test corresponds to 188 # test_executable_api.md#filtering-tests-on-the-command-line 189 # and 190 # test_executable_api.md#using-a-filter-file 191 s = _SetOfTestFiltersGroups([self._filter1]) 192 self.assertEqual(['Foo.Bar.bar1', 'Foo.Bar.bar2'], 193 s.filter_test_names(self._tests)) 194 195 def test_combining_multiple_filters1(self): 196 # This test corresponds to the first test under 197 # test_executable_api.md#combining-multiple-filters 198 s = _SetOfTestFiltersGroups([ 199 _TestFiltersGroup.from_string('Foo.Bar.*'), 200 _TestFiltersGroup.from_string('Foo.Bar.bar2') 201 ]) 202 self.assertEqual(['Foo.Bar.bar2'], s.filter_test_names(self._tests)) 203 204 def test_combining_multiple_filters2(self): 205 # This test corresponds to the second test under 206 # test_executable_api.md#combining-multiple-filters 207 # TODO([email protected]): Figure out if the 3rd test example from 208 # the docs has correct inputs+outputs (or if there are some typos). 209 s = _SetOfTestFiltersGroups([ 210 _TestFiltersGroup.from_string('Foo.Bar.*'), 211 _TestFiltersGroup.from_string('Foo.Baz.baz') 212 ]) 213 self.assertEqual([], s.filter_test_names(self._tests)) 214 215 216class PublicApiTests(fake_filesystem_unittest.TestCase): 217 def test_filter_cmdline_arg(self): 218 parser = argparse.ArgumentParser() 219 test_filtering.add_cmdline_args(parser) 220 args = parser.parse_args(args=[ 221 '--isolated-script-test-filter=-barbaz', 222 '--isolated-script-test-filter=foo*::bar*' 223 ]) 224 self.assertEqual( 225 ['foo1', 'foo2', 'bar1', 'bar2'], 226 test_filtering.filter_tests( 227 args, {}, ['foo1', 'foo2', 'bar1', 'bar2', 'barbaz', 'zzz'])) 228 229 def test_filter_file_cmdline_arg(self): 230 # pylint: disable=unexpected-keyword-arg 231 f = tempfile.NamedTemporaryFile(delete=False, 232 mode='w', 233 encoding='utf-8') 234 try: 235 filepath = f.name 236 f.write('foo*') 237 f.close() 238 239 parser = argparse.ArgumentParser() 240 test_filtering.add_cmdline_args(parser) 241 args = parser.parse_args(args=[ 242 '--isolated-script-test-filter-file={0:s}'.format(filepath) 243 ]) 244 self.assertEqual(['foo1', 'foo2'], 245 test_filtering.filter_tests( 246 args, {}, ['foo1', 'foo2', 'bar1', 'bar2'])) 247 finally: 248 os.remove(filepath) 249 250 251def _shard_tests(input_test_list_string, input_env): 252 input_test_list = input_test_list_string.split(',') 253 output_test_list = test_filtering._shard_tests(input_test_list, input_env) 254 return ','.join(output_test_list) 255 256 257class ShardingTest(unittest.TestCase): 258 def test_empty_environment(self): 259 self.assertEqual('a,b,c', _shard_tests('a,b,c', {})) 260 261 def test_basic_sharding(self): 262 self.assertEqual( 263 'a,c,e', 264 _shard_tests('a,b,c,d,e', { 265 'GTEST_SHARD_INDEX': '0', 266 'GTEST_TOTAL_SHARDS': '2' 267 })) 268 self.assertEqual( 269 'b,d', 270 _shard_tests('a,b,c,d,e', { 271 'GTEST_SHARD_INDEX': '1', 272 'GTEST_TOTAL_SHARDS': '2' 273 })) 274 275 def test_error_conditions(self): 276 # shard index > total shards 277 with self.assertRaises(Exception): 278 _shard_tests('', { 279 'GTEST_SHARD_INDEX': '2', 280 'GTEST_TOTAL_SHARDS': '2' 281 }) 282 283 # non-integer shard index 284 with self.assertRaises(Exception): 285 _shard_tests('', { 286 'GTEST_SHARD_INDEX': 'a', 287 'GTEST_TOTAL_SHARDS': '2' 288 }) 289 290 # non-integer total shards 291 with self.assertRaises(Exception): 292 _shard_tests('', { 293 'GTEST_SHARD_INDEX': '0', 294 'GTEST_TOTAL_SHARDS': 'b' 295 }) 296 297 298if __name__ == '__main__': 299 unittest.main() 300