1# 2# Copyright (C) 2020 The Android Open Source Project 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15# 16 17import copy 18import logging 19import re 20from sre_constants import error as regex_error 21import types 22from typing import Set, List, Tuple, Callable, Union, Iterable 23 24from host import const 25from common import list_utils 26 27REGEX_PREFIX = 'r(' 28REGEX_SUFFIX = ')' 29REGEX_PREFIX_ESCAPE = '\\r(' 30NEGATIVE_PATTERN_PREFIX = '-' 31_INCLUDE_FILTER = '_include_filter' 32_EXCLUDE_FILTER = '_exclude_filter' 33DEFAULT_EXCLUDE_OVER_INCLUDE = False 34_MODULE_NAME_PATTERN = '{module}.{test}' 35 36 37def ExpandBitness(input_list: Iterable[str]) -> Set[str]: 38 '''Expand filter items with bitness suffix. 39 40 If a filter item contains bitness suffix, only test name with that tag 41 will be included in output. 42 Otherwise, 2 more item with 32bit and 64bit suffix will be added to the output list. 43 44 This method removes duplicated item before returning output. 45 46 Examples of input -> output are: 47 [a_32bit] -> [a_32bit] 48 [a] -> [a, a_32bit, a_64bit] 49 [a_32bit, a] -> [a_32bit, a, a_64bit] 50 51 Args: 52 input_list: set of string, the set to expand 53 54 Returns: 55 A set of string 56 ''' 57 result = set() 58 for item in input_list: 59 result.add(str(item)) 60 if (not item.endswith(const.SUFFIX_32BIT) and 61 not item.endswith(const.SUFFIX_64BIT)): 62 result.add("%s_%s" % (item, const.SUFFIX_32BIT)) 63 result.add("%s_%s" % (item, const.SUFFIX_64BIT)) 64 return result 65 66 67def ExpandAppendix(input_list: Iterable[str], appendix_list: Iterable[str], filter_pattern: str) -> Set[str]: 68 '''Expand each item in input_list with appendix in the appendix_list 69 70 For each item in input_list, expand it to N items (N=size of appendix_list) 71 by attaching it with each appendix form appendix_list. 72 Note, for items end with bitness info (e.g 32bit/63bit/_32bit/_64bit), 73 attach the appendix before the bitness info. (This is to make sure the 74 bitness info is always at the end of each item since the system rely on this 75 assumption to check the bitness info) 76 There are two cases when an item will not be expanded: 1) it is a Regex 77 filter item and 2) it has the pattern described by filter_pattern. 78 79 Examples of input -> output are: 80 [a] [_default] -> [a_default] 81 [a, b] [_default] -> [a_default, b_default] 82 [a] [_default, _test] -> [a_default, a_test] 83 [a, b_32bit] [_default, _test] 84 -> [a_default, a_test, b_default_32bit, b_test_32bit] 85 86 Args: 87 input_list: set of string, the list to expand 88 appendix_list: set of string, the appendix to be append. 89 filter_pattern: string, a Regex pattern to filter out the items that 90 should not expand. 91 92 Returns: 93 A set of string with expanded result. 94 ''' 95 result = set() 96 for item in input_list: 97 if IsRegexFilter(item) or re.compile(filter_pattern).match(item): 98 result.add(item) 99 continue 100 pos = len(item) 101 if (item.endswith(const.SUFFIX_32BIT) or 102 item.endswith(const.SUFFIX_64BIT)): 103 pos = len(item) - len(const.SUFFIX_32BIT) 104 if item[pos - 1] == "_": 105 pos = pos - 1 106 for appendix in appendix_list: 107 result.add(item[:pos] + appendix + item[pos:]) 108 return result 109 110 111def SplitFilterList(input_list: Iterable[str]) -> Tuple[Set[str], List[str]]: 112 '''Split filter items into exact and regex lists. 113 114 To specify a regex filter, the syntax is: 115 'r(suite.test)' for regex matching of 'suite.test', where '.' means 116 one of any char. 117 See Filter class docstring for details. 118 119 Args: 120 input_list: list of string, the list to split 121 122 Returns: 123 A tuple of lists: two lists where the first one is exact matching 124 list and second one is regex list where the wrapping 125 syntax 'r(..)' is removed. 126 ''' 127 exact = set() 128 regex = [] 129 for item in input_list: 130 if IsRegexFilter(item): 131 regex_item = item[len(REGEX_PREFIX):-len(REGEX_SUFFIX)] 132 try: 133 re.compile(regex_item) 134 regex.append(regex_item) 135 except regex_error: 136 logging.error('Invalid regex %s, ignored. Please refer to ' 137 'python re syntax documentation.' % regex_item) 138 elif item.startswith(REGEX_PREFIX_ESCAPE) and item.endswith( 139 REGEX_SUFFIX): 140 exact.add(REGEX_PREFIX + item[len(REGEX_PREFIX_ESCAPE):]) 141 else: 142 exact.add(item) 143 144 return (exact, regex) 145 146 147def SplitNegativePattern(input_list: Iterable[str]) -> Tuple[Set[str], Set[str]]: 148 '''Split negative items out from an input filter list. 149 150 Items starting with the negative sign will be moved to the second returning 151 list. 152 153 Args: 154 input_list: list of string, the list to split 155 156 Returns: 157 A tuple of lists: two lists where the first one is positive patterns 158 and second one is negative items whose negative sign 159 is removed. 160 ''' 161 positive = set() 162 negative = set() 163 for item in input_list: 164 if item.startswith(NEGATIVE_PATTERN_PREFIX): 165 negative.add(item[len(NEGATIVE_PATTERN_PREFIX):]) 166 else: 167 positive.add(item) 168 return (positive, negative) 169 170 171def InRegexList(item: str, regex_list: List[str]) -> bool: 172 '''Checks whether a given string matches an item in the given regex list. 173 174 Args: 175 item: string, given string 176 regex_list: regex list 177 178 Returns: 179 bool, True if there is a match; False otherwise. 180 ''' 181 for regex in regex_list: 182 p = re.compile(regex) 183 m = p.match(item) 184 if m and m.start() == 0 and m.end() == len(item): 185 return True 186 187 return False 188 189 190def IsRegexFilter(item: str) -> bool: 191 '''Checks whether the given item is a regex filter. 192 193 Args: 194 item: string, given string 195 196 Returns: 197 bool: true if the given item is a regex filter. 198 ''' 199 return item.startswith(REGEX_PREFIX) and item.endswith(REGEX_SUFFIX) 200 201 202class Filter(object): 203 '''A class to hold test filter rules and filter test names. 204 205 Regex matching is supported. Regex syntax is python re package syntax. 206 To specify a regex filter, the syntax is: 207 'suite.test' for exact matching 208 'r(suite.test)' for regex matching of 'suite.test', where '.' means 209 one of any char. 210 '\r(suite.test)' for exact matching of name 'r(suite.test)', where 211 '\r' is a two char string ('\\r' in code). 212 Since test name is not expected to start with backslash, the exact 213 string matching of name '\r(suite.test)' is not supported here. 214 215 Negative pattern is supported. If a test name starts with the negative 216 sign in include_filter, the negative sign will be removed and item will 217 be moved from include_filter to exclude_filter. Negative sign should 218 be added before regex prefix, i.e., '-r(negative.pattern)' 219 220 Attributes: 221 enable_regex: bool, whether regex is enabled. 222 include_filter: set of string, input include filter 223 exclude_filter: set of string, input exclude filter 224 include_filter_exact: set of string, exact include filter 225 include_filter_regex: list of string, exact include filter 226 exclude_filter_exact: set of string, exact exclude filter 227 exclude_filter_regex: list of string, exact exclude filter 228 exclude_over_include: bool, False for include over exclude; 229 True for exclude over include. 230 enable_native_pattern: bool, whether to enable negative pattern 231 processing in include_filter 232 enable_module_name_prefix_matching: bool, whether to perform auto 233 module name prefix matching 234 module_name: string, test module name for auto module name prefix 235 matching 236 expand_bitness: bool, whether to append bitness to filter items. 237 Default is False. When set to True, bitness will 238 be added to test name for filtering process, but 239 the original filter list will not be changed. 240 ''' 241 include_filter_exact = set() 242 include_filter_regex = [] 243 exclude_filter_exact = set() 244 exclude_filter_regex = [] 245 246 def __init__(self, 247 include_filter=set(), 248 exclude_filter=set(), 249 enable_regex=True, 250 exclude_over_include=None, 251 enable_negative_pattern=True, 252 enable_module_name_prefix_matching=False, 253 module_name=None, 254 expand_bitness=False): 255 self.enable_regex = enable_regex 256 self.expand_bitness = expand_bitness 257 258 self.enable_negative_pattern = enable_negative_pattern 259 self.include_filter = include_filter 260 self.exclude_filter = exclude_filter 261 if exclude_over_include is None: 262 exclude_over_include = DEFAULT_EXCLUDE_OVER_INCLUDE 263 self.exclude_over_include = exclude_over_include 264 self.enable_module_name_prefix_matching = enable_module_name_prefix_matching 265 self.module_name = module_name 266 267 # @Deprecated. Use expand_bitness parameter in construction method instead. 268 # This method will be removed after all legacy usage has been cleaned up 269 def ExpandBitness(self): 270 '''Expand bitness from filter. 271 272 Items in the filter that doesn't contain bitness suffix will be expended 273 to 3 items, 2 of which ending with bitness. This method is safe if 274 called multiple times. Regex items will not be expanded 275 ''' 276 self.include_filter_exact = ExpandBitness(self.include_filter_exact) 277 self.exclude_filter_exact = ExpandBitness(self.exclude_filter_exact) 278 self.expand_bitness = True 279 280 def IsIncludeFilterEmpty(self): 281 '''Check whether actual include filter is specified. 282 283 Since the input include filter may contain negative patterns, 284 checking self.include_filter is not always correct. 285 286 This method checks include_filter_exact and include_filter_regex. 287 ''' 288 return not self.include_filter_exact and not self.include_filter_regex 289 290 def ExpandAppendix(self, appendix_list: Iterable[str], filter_pattern: str): 291 '''Expand filter with appendix from appendix_list. 292 293 Reset both include_filter and exclude_filter by expanding the filters 294 with appendix in appendix_list. 295 296 Args: 297 appendix_list: list of string to be append to the filters. 298 filter_pattern: string, a Regex pattern to filter out the items that 299 should not be expanded. 300 ''' 301 self.include_filter = ExpandAppendix(self.include_filter, 302 appendix_list, filter_pattern) 303 self.exclude_filter = ExpandAppendix(self.exclude_filter, 304 appendix_list, filter_pattern) 305 306 def Filter(self, item: str) -> bool: 307 '''Filter a given string using the internal filters. 308 309 Rule: 310 By default, include_filter overrides exclude_filter. This means: 311 If include_filter is empty, only exclude_filter is checked. 312 Otherwise, only include_filter is checked 313 If exclude_over_include is set to True, exclude filter will first 314 be checked. 315 316 Args: 317 item: string, the string for filter check 318 319 Returns: 320 bool. True if it passed the filter; False otherwise 321 ''' 322 if self.exclude_over_include: 323 if self.IsInExcludeFilter(item): 324 return False 325 326 if not self.IsIncludeFilterEmpty(): 327 return self.IsInIncludeFilter(item) 328 329 return True 330 else: 331 if not self.IsIncludeFilterEmpty(): 332 return self.IsInIncludeFilter(item) 333 334 return not self.IsInExcludeFilter(item) 335 336 def IsInIncludeFilter(self, item: str) -> bool: 337 '''Check if item is in include filter. 338 339 If enable_module_name_prefix_matching is set to True, module name 340 added to item as prefix will also be check from the include filter. 341 342 Args: 343 item: string, item to check filter 344 345 Returns: 346 bool, True if in include filter. 347 ''' 348 return self._ModuleNamePrefixMatchingCheck(item, 349 self._IsInIncludeFilter) 350 351 def IsInExcludeFilter(self, item: str) -> bool: 352 '''Check if item is in exclude filter. 353 354 If enable_module_name_prefix_matching is set to True, module name 355 added to item as prefix will also be check from the exclude filter. 356 357 Args: 358 item: string, item to check filter 359 360 Returns: 361 bool, True if in exclude filter. 362 ''' 363 return self._ModuleNamePrefixMatchingCheck(item, 364 self._IsInExcludeFilter) 365 366 def _ModuleNamePrefixMatchingCheck(self, item: str, check_function: Callable) -> bool: 367 '''Check item from filter after appending module name as prefix. 368 369 This function will first check whether enable_module_name_prefix_matching 370 is True and module_name is not empty. Then, the check_function will 371 be applied to the item. If the result is False and 372 enable_module_name_prefix_matching is True, module name will be added 373 as the prefix to the item, in format of '<module_name>.<item>', and 374 call the check_function again with the new resulting name. 375 376 This is mainly used for retry command where test module name are 377 automatically added to test case name. 378 379 Args: 380 item: string, test name for checking. 381 check_function: function to check item in filters. 382 383 Return: 384 bool, True if item pass the filter from the given check_function. 385 ''' 386 res = check_function(item) 387 388 if (not res and self.enable_module_name_prefix_matching and 389 self.module_name): 390 res = check_function( 391 _MODULE_NAME_PATTERN.format( 392 module=self.module_name, test=item)) 393 394 return res 395 396 def _IsInIncludeFilter(self, item: str) -> bool: 397 '''Internal function to check if item is in include filter. 398 399 Args: 400 item: string, item to check filter 401 402 Returns: 403 bool, True if in include filter. 404 ''' 405 return item in self.include_filter_exact or InRegexList( 406 item, self.include_filter_regex) 407 408 def _IsInExcludeFilter(self, item: str) -> bool: 409 '''Internal function to check if item is in exclude filter. 410 411 Args: 412 item: string, item to check filter 413 414 Returns: 415 bool, True if in exclude filter. 416 ''' 417 return item in self.exclude_filter_exact or InRegexList( 418 item, self.exclude_filter_regex) 419 420 @property 421 def include_filter(self): 422 '''Getter method for include_filter. 423 424 Use this method to print include_filter only. 425 426 If the items needed to be added, use add_to_exclude_filter method. 427 428 E.g. 429 self.add_to_exclude_filter('pattern1') 430 431 If the filter needs to be modified without using add_to_exclude_filter, 432 call refresh_filter() after modification. Otherwise, the change will 433 not take effect. 434 435 E.g. 436 Get and modify filter: 437 filter = Filter() 438 filter.include_filter.append('pattern1') 439 Refresh the filter: 440 filter.refresh_filter() 441 ''' 442 return getattr(self, _INCLUDE_FILTER, set()) 443 444 @include_filter.setter 445 def include_filter(self, include_filter: Set[str]): 446 '''Setter method for include_filter''' 447 setattr(self, _INCLUDE_FILTER, include_filter) 448 self.refresh_filter() 449 450 @property 451 def exclude_filter(self): 452 '''Getter method for exclude_filter. 453 454 Use this method to print exclude_filter only. 455 456 If the items needed to be added, use add_to_exclude_filter method. 457 458 E.g. 459 self.add_to_exclude_filter('pattern1') 460 461 If the filter needs to be modified without using add_to_exclude_filter, 462 call refresh_filter() after modification. Otherwise, the change will 463 not take effect. 464 465 E.g. 466 Get and modify filter: 467 filter = Filter() 468 filter.exclude_filter.append('pattern1') 469 Refresh the filter: 470 filter.refresh_filter() 471 ''' 472 return getattr(self, _EXCLUDE_FILTER, set()) 473 474 @exclude_filter.setter 475 def exclude_filter(self, exclude_filter: Set[str]): 476 '''Setter method for exclude_filter''' 477 setattr(self, _EXCLUDE_FILTER, exclude_filter) 478 self.refresh_filter() 479 480 def add_to_include_filter(self, pattern: Union[str, Set[str]], auto_refresh: bool = True): 481 '''Add an item to include_filter. 482 483 Args: 484 pattern: string or set of string. Item(s) to add 485 auto_refresh: bool, whether to automatically call refresh_filter(). 486 Default is True. Use False only if a large number of 487 items are added one by one in a sequence call. 488 In that case, call refresh_filter() at the end of the 489 sequence. 490 ''' 491 if not isinstance(pattern, set): 492 pattern = {pattern} 493 494 self.include_filter.update(pattern) 495 496 if auto_refresh: 497 self.refresh_filter() 498 499 def add_to_exclude_filter(self, pattern: Union[str, Set[str]], auto_refresh: bool = True): 500 '''Add an item to exclude_filter. 501 502 Args: 503 pattern: string or set of string. Item(s) to add 504 auto_refresh: bool, whether to automatically call refresh_filter(). 505 Default is True. Use False only if a large number of 506 items are added one by one in a sequence call. 507 In that case, call refresh_filter() at the end of the 508 sequence. 509 ''' 510 if not isinstance(pattern, set): 511 pattern = {pattern} 512 513 self.exclude_filter.update(pattern) 514 515 if auto_refresh: 516 self.refresh_filter() 517 518 def refresh_filter(self): 519 '''Process the filter patterns. 520 521 This method splits filter into exact and regex patterns. 522 Bitness will also be appended if expand_bitness is True. 523 ''' 524 include_filter = copy.copy(self.include_filter) 525 exclude_filter = copy.copy(self.exclude_filter) 526 527 if self.enable_negative_pattern: 528 include_filter, include_filter_negative = SplitNegativePattern( 529 include_filter) 530 exclude_filter.update(include_filter_negative) 531 532 if self.enable_regex: 533 self.include_filter_exact, self.include_filter_regex = SplitFilterList( 534 include_filter) 535 self.exclude_filter_exact, self.exclude_filter_regex = SplitFilterList( 536 exclude_filter) 537 else: 538 self.include_filter_exact = include_filter 539 self.exclude_filter_exact = exclude_filter 540 541 if self.expand_bitness: 542 self.include_filter_exact = ExpandBitness( 543 self.include_filter_exact) 544 self.exclude_filter_exact = ExpandBitness( 545 self.exclude_filter_exact) 546 547 def __str__(self): 548 return ('Filter:\nenable_regex: {enable_regex}\n' 549 'enable_negative_pattern: {enable_negative_pattern}\n' 550 'enable_module_name_prefix_matching: ' 551 '{enable_module_name_prefix_matching}\n' 552 'module_name: {module_name}\n' 553 'include_filter: {include_filter}\n' 554 'exclude_filter: {exclude_filter}\n' 555 'include_filter_exact: {include_filter_exact}\n' 556 'include_filter_regex: {include_filter_regex}\n' 557 'exclude_filter_exact: {exclude_filter_exact}\n' 558 'exclude_filter_regex: {exclude_filter_regex}\n' 559 'expand_bitness: {expand_bitness}'.format( 560 enable_regex=self.enable_regex, 561 enable_negative_pattern=self.enable_negative_pattern, 562 enable_module_name_prefix_matching= 563 self.enable_module_name_prefix_matching, 564 module_name=self.module_name, 565 include_filter=self.include_filter, 566 exclude_filter=self.exclude_filter, 567 include_filter_exact=self.include_filter_exact, 568 include_filter_regex=self.include_filter_regex, 569 exclude_filter_exact=self.exclude_filter_exact, 570 exclude_filter_regex=self.exclude_filter_regex, 571 expand_bitness=self.expand_bitness)) 572