1# Copyright 2024, The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15""" 16Simple parsing code to scan test_mapping files and determine which 17modules are needed to build for the given list of changed files. 18TODO(lucafarsi): Deduplicate from artifact_helper.py 19""" 20# TODO(lucafarsi): Share this logic with the original logic in 21# test_mapping_test_retriever.py 22 23import json 24import os 25import re 26from typing import Any 27 28# Regex to extra test name from the path of test config file. 29TEST_NAME_REGEX = r'(?:^|.*/)([^/]+)\.config' 30 31# Key name for TEST_MAPPING imports 32KEY_IMPORTS = 'imports' 33KEY_IMPORT_PATH = 'path' 34 35# Name of TEST_MAPPING file. 36TEST_MAPPING = 'TEST_MAPPING' 37 38# Pattern used to identify double-quoted strings and '//'-format comments in 39# TEST_MAPPING file, but only double-quoted strings are included within the 40# matching group. 41_COMMENTS_RE = re.compile(r'(\"(?:[^\"\\]|\\.)*\"|(?=//))(?://.*)?') 42 43 44def FilterComments(test_mapping_file: str) -> str: 45 """Remove comments in TEST_MAPPING file to valid format. 46 47 Only '//' is regarded as comments. 48 49 Args: 50 test_mapping_file: Path to a TEST_MAPPING file. 51 52 Returns: 53 Valid json string without comments. 54 """ 55 return re.sub(_COMMENTS_RE, r'\1', test_mapping_file) 56 57def GetTestMappings(paths: set[str], 58 checked_paths: set[str]) -> dict[str, dict[str, Any]]: 59 """Get the affected TEST_MAPPING files. 60 61 TEST_MAPPING files in source code are packaged into a build artifact 62 `test_mappings.zip`. Inside the zip file, the path of each TEST_MAPPING file 63 is preserved. From all TEST_MAPPING files in the source code, this method 64 locates the affected TEST_MAPPING files based on the given paths list. 65 66 A TEST_MAPPING file may also contain `imports` that import TEST_MAPPING files 67 from a different location, e.g., 68 "imports": [ 69 { 70 "path": "../folder2" 71 } 72 ] 73 In that example, TEST_MAPPING files inside ../folder2 (relative to the 74 TEST_MAPPING file containing that imports section) and its parent directories 75 will also be included. 76 77 Args: 78 paths: A set of paths with related TEST_MAPPING files for given changes. 79 checked_paths: A set of paths that have been checked for TEST_MAPPING file 80 already. The set is updated after processing each TEST_MAPPING file. It's 81 used to prevent infinite loop when the method is called recursively. 82 83 Returns: 84 A dictionary of Test Mapping containing the content of the affected 85 TEST_MAPPING files, indexed by the path containing the TEST_MAPPING file. 86 """ 87 test_mappings = {} 88 89 # Search for TEST_MAPPING files in each modified path and its parent 90 # directories. 91 all_paths = set() 92 for path in paths: 93 dir_names = path.split(os.path.sep) 94 all_paths |= set( 95 [os.path.sep.join(dir_names[:i + 1]) for i in range(len(dir_names))]) 96 # Add root directory to the paths to search for TEST_MAPPING file. 97 all_paths.add('') 98 99 all_paths.difference_update(checked_paths) 100 checked_paths |= all_paths 101 # Try to load TEST_MAPPING file in each possible path. 102 for path in all_paths: 103 try: 104 test_mapping_file = os.path.join(os.path.join(os.getcwd(), path), 'TEST_MAPPING') 105 # Read content of TEST_MAPPING file. 106 content = FilterComments(open(test_mapping_file, "r").read()) 107 test_mapping = json.loads(content) 108 test_mappings[path] = test_mapping 109 110 import_paths = set() 111 for import_detail in test_mapping.get(KEY_IMPORTS, []): 112 import_path = import_detail[KEY_IMPORT_PATH] 113 # Try the import path as absolute path. 114 import_paths.add(import_path) 115 # Try the import path as relative path based on the test mapping file 116 # containing the import. 117 norm_import_path = os.path.normpath(os.path.join(path, import_path)) 118 import_paths.add(norm_import_path) 119 import_paths.difference_update(checked_paths) 120 if import_paths: 121 import_test_mappings = GetTestMappings(import_paths, checked_paths) 122 test_mappings.update(import_test_mappings) 123 except (KeyError, FileNotFoundError, NotADirectoryError): 124 # TEST_MAPPING file doesn't exist in path 125 pass 126 127 return test_mappings 128 129 130def FindAffectedModules( 131 test_mappings: dict[str, Any], 132 changed_files: set[str], 133 test_mapping_test_groups: set[str], 134) -> set[str]: 135 """Find affected test modules. 136 137 Find the affected set of test modules that would run in a test mapping run based on the given test mappings, changed files, and test mapping test group. 138 139 Args: 140 test_mappings: A set of test mappings returned by GetTestMappings in the following format: 141 { 142 'test_mapping_file_path': { 143 'group_name' : [ 144 'name': 'module_name', 145 ], 146 } 147 } 148 changed_files: A set of files changed for the given run. 149 test_mapping_test_groups: A set of test mapping test groups that are being considered for the given run. 150 151 Returns: 152 A set of test module names which would run for a test mapping test run with the given parameters. 153 """ 154 155 modules = set() 156 157 for test_mapping in test_mappings.values(): 158 for group_name, group in test_mapping.items(): 159 # If a module is not in any of the test mapping groups being tested skip 160 # it. 161 if group_name not in test_mapping_test_groups: 162 continue 163 164 for entry in group: 165 module_name = entry.get('name') 166 167 if not module_name: 168 continue 169 170 file_patterns = entry.get('file_patterns') 171 if not file_patterns: 172 modules.add(module_name) 173 continue 174 175 if matches_file_patterns(file_patterns, changed_files): 176 modules.add(module_name) 177 178 return modules 179 180def MatchesFilePatterns( 181 file_patterns: list[set], changed_files: set[str] 182) -> bool: 183 """Checks if any of the changed files match any of the file patterns. 184 185 Args: 186 file_patterns: A list of file patterns to match against. 187 changed_files: A set of files to check against the file patterns. 188 189 Returns: 190 True if any of the changed files match any of the file patterns. 191 """ 192 return any(re.search(pattern, "|".join(changed_files)) for pattern in file_patterns) 193