1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Executes a compilation failure test.""" 15 16import argparse 17import logging 18from pathlib import Path 19import re 20import shlex 21import string 22import sys 23import subprocess 24 25import pw_cli.log 26 27from pw_compilation_testing.generator import Compiler, Expectation, TestCase 28 29_LOG = logging.getLogger(__package__) 30 31_RULE_REGEX = re.compile('^rule (?:cxx|.*_cxx)$') 32_NINJA_VARIABLE = re.compile('^([a-zA-Z0-9_]+) = ?') 33 34 35# TODO(hepler): Could do this step just once and output the results. 36def find_cc_rule(toolchain_ninja_file: Path) -> str | None: 37 """Searches the toolchain.ninja file for the cc rule.""" 38 cmd_prefix = ' command = ' 39 40 found_rule = False 41 42 with toolchain_ninja_file.open() as fd: 43 for line in fd: 44 if found_rule: 45 if line.startswith(cmd_prefix): 46 cmd = line[len(cmd_prefix) :].strip() 47 if cmd.startswith('ccache '): 48 cmd = cmd[len('ccache ') :] 49 return cmd 50 51 if not line.startswith(' '): 52 break 53 elif _RULE_REGEX.match(line): 54 found_rule = True 55 56 return None 57 58 59def _parse_ninja_variables(target_ninja_file: Path) -> dict[str, str]: 60 variables: dict[str, str] = {} 61 62 with target_ninja_file.open() as fd: 63 for line in fd: 64 match = _NINJA_VARIABLE.match(line) 65 if match: 66 variables[match.group(1)] = line[match.end() :].strip() 67 68 return variables 69 70 71_EXPECTED_GN_VARS = ( 72 'asmflags', 73 'cflags', 74 'cflags_c', 75 'cflags_cc', 76 'cflags_objc', 77 'cflags_objcc', 78 'defines', 79 'include_dirs', 80) 81 82_TEST_MACRO = 'PW_NC_TEST_EXECUTE_CASE_' 83# Regular expression to find and remove ANSI escape sequences, based on 84# https://stackoverflow.com/a/14693789. 85_ANSI_ESCAPE_SEQUENCES = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') 86 87 88class _TestFailure(Exception): 89 pass 90 91 92def _red(message: str) -> str: 93 return f'\033[31m\033[1m{message}\033[0m' 94 95 96_TITLE_1 = ' NEGATIVE ' 97_TITLE_2 = ' COMPILATION TEST ' 98 99_BOX_TOP = f'┏{"━" * len(_TITLE_1)}┓' 100_BOX_MID_1 = f'┃{_red(_TITLE_1)}┃ \033[1m{{test_name}}\033[0m' 101_BOX_MID_2 = f'┃{_red(_TITLE_2)}┃ \033[0m{{source}}:{{line}}\033[0m' 102_BOX_BOT = f'┻{"━" * (len(_TITLE_1))}┻{"━" * (77 - len(_TITLE_1))}┓' 103_FOOTER = '\n' + '━' * 79 + '┛' 104 105 106def _start_failure(test: TestCase, command: str) -> None: 107 print(_BOX_TOP, file=sys.stderr) 108 print(_BOX_MID_1.format(test_name=test.name()), file=sys.stderr) 109 print( 110 _BOX_MID_2.format(source=test.source, line=test.line), file=sys.stderr 111 ) 112 print(_BOX_BOT, file=sys.stderr) 113 print(file=sys.stderr) 114 115 _LOG.debug('Compilation command:\n%s', command) 116 117 118def _check_results( 119 test: TestCase, command: str, process: subprocess.CompletedProcess 120) -> None: 121 stderr = process.stderr.decode(errors='replace') 122 123 if process.returncode == 0: 124 _start_failure(test, command) 125 _LOG.error('Compilation succeeded, but it should have failed!') 126 _LOG.error('Update the test code so that is fails to compile.') 127 _LOG.error('Compilation command:\n%s', command) 128 raise _TestFailure 129 130 compiler_str = command.split(' ', 1)[0] 131 compiler = Compiler.from_command(compiler_str) 132 133 _LOG.debug('%s is %s', compiler_str, compiler) 134 expectations: list[Expectation] = [ 135 e for e in test.expectations if compiler.matches(e.compiler) 136 ] 137 138 _LOG.debug( 139 '%s: Checking compilation from %s (%s) for %d of %d patterns:', 140 test.name(), 141 compiler_str, 142 compiler, 143 len(expectations), 144 len(test.expectations), 145 ) 146 for expectation in expectations: 147 _LOG.debug(' %s', expectation.pattern.pattern) 148 149 if not expectations: 150 _start_failure(test, command) 151 _LOG.error( 152 'Compilation with %s failed, but no PW_NC_EXPECT() patterns ' 153 'that apply to %s were provided', 154 compiler_str, 155 compiler_str, 156 ) 157 158 _LOG.error('Compilation output:\n%s', stderr) 159 _LOG.error('') 160 _LOG.error( 161 'Add at least one PW_NC_EXPECT("<regex>") or ' 162 'PW_NC_EXPECT_%s("<regex>") expectation to %s', 163 compiler.name, 164 test.case, 165 ) 166 raise _TestFailure 167 168 no_color = _ANSI_ESCAPE_SEQUENCES.sub('', stderr) 169 170 failed = [e for e in expectations if not e.pattern.search(no_color)] 171 if failed: 172 _start_failure(test, command) 173 _LOG.error( 174 'Compilation with %s failed, but the output did not ' 175 'match the expected patterns.', 176 compiler_str, 177 ) 178 _LOG.error( 179 '%d of %d expected patterns did not match:', 180 len(failed), 181 len(expectations), 182 ) 183 _LOG.error('') 184 for expectation in expectations: 185 _LOG.error( 186 ' %s %s:%d: %s', 187 '❌' if expectation in failed else '✅', 188 test.source.name, 189 expectation.line, 190 expectation.pattern.pattern, 191 ) 192 _LOG.error('') 193 194 _LOG.error('Compilation output:\n%s', stderr) 195 _LOG.error('') 196 _LOG.error( 197 'Update the test so that compilation fails with the ' 198 'expected output' 199 ) 200 raise _TestFailure 201 202 203def _should_skip_test(base_command: str) -> bool: 204 # Attempt to run the preprocessor while setting the PW_NC_TEST macro to an 205 # illegal statement (defined() with no identifier). If the preprocessor 206 # passes, the test was skipped. 207 preprocessor_cmd = f'{base_command}{shlex.quote("defined()")} -E' 208 209 split_cmd = shlex.split(preprocessor_cmd) 210 if "-c" in split_cmd: 211 split_cmd.remove("-c") 212 preprocessor_cmd = shlex.join(split_cmd) 213 214 process = subprocess.run( 215 preprocessor_cmd, 216 shell=True, 217 stdout=subprocess.DEVNULL, 218 stderr=subprocess.DEVNULL, 219 ) 220 _LOG.debug( 221 'Preprocessor command to check if test is enabled returned %d:\n%s', 222 process.returncode, 223 preprocessor_cmd, 224 ) 225 return process.returncode == 0 226 227 228def _execute_test( 229 test: TestCase, 230 command: str, 231 variables: dict[str, str], 232 all_tests: list[str], 233) -> None: 234 variables['in'] = str(test.source) 235 236 base_command = ' '.join( 237 [ 238 string.Template(command).substitute(variables), 239 '-DPW_NEGATIVE_COMPILATION_TESTS_ENABLED', 240 # Define macros to disable all tests except this one. 241 *(f'-D{_TEST_MACRO}{t}=0' for t in all_tests if t != test.case), 242 f'-D{_TEST_MACRO}{test.case}=', 243 ] 244 ) 245 246 if _should_skip_test(base_command): 247 _LOG.info( 248 "Skipping test %s since it is excluded by the preprocessor", 249 test.name(), 250 ) 251 return 252 253 compile_command = base_command + '1' # set macro to 1 to enable the test 254 process = subprocess.run(compile_command, shell=True, capture_output=True) 255 _check_results(test, compile_command, process) 256 257 258def _main( 259 test: TestCase, toolchain_ninja: Path, target_ninja: Path, all_tests: Path 260) -> int: 261 """Compiles a compile fail test and returns 1 if compilation succeeds.""" 262 command = find_cc_rule(toolchain_ninja) 263 264 if command is None: 265 _LOG.critical( 266 'Failed to find C++ compilation command in %s', toolchain_ninja 267 ) 268 return 2 269 270 variables = {key: '' for key in _EXPECTED_GN_VARS} 271 variables.update(_parse_ninja_variables(target_ninja)) 272 273 variables['out'] = str( 274 target_ninja.parent / f'{target_ninja.stem}.compile_fail_test.out' 275 ) 276 277 try: 278 _execute_test( 279 test, command, variables, all_tests.read_text().splitlines() 280 ) 281 except _TestFailure: 282 print(_FOOTER, file=sys.stderr) 283 return 1 284 285 return 0 286 287 288def _parse_args() -> dict: 289 """Parses command-line arguments.""" 290 291 parser = argparse.ArgumentParser( 292 description='Emits an error when a facade has a null backend' 293 ) 294 parser.add_argument( 295 '--toolchain-ninja', 296 type=Path, 297 required=True, 298 help='Ninja file with the compilation command for the toolchain', 299 ) 300 parser.add_argument( 301 '--target-ninja', 302 type=Path, 303 required=True, 304 help='Ninja file with the compilation commands to the test target', 305 ) 306 parser.add_argument( 307 '--test-data', 308 dest='test', 309 required=True, 310 type=TestCase.deserialize, 311 help='Serialized TestCase object', 312 ) 313 parser.add_argument('--all-tests', type=Path, help='List of all tests') 314 return vars(parser.parse_args()) 315 316 317if __name__ == '__main__': 318 pw_cli.log.install(level=logging.INFO) 319 sys.exit(_main(**_parse_args())) 320