1#!/usr/bin/env python 2# Copyright 2022 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Watch build config dataclasses.""" 16 17from __future__ import annotations 18 19from dataclasses import dataclass, field 20import functools 21import logging 22from pathlib import Path 23import shlex 24from typing import Callable, Mapping, TYPE_CHECKING 25 26from prompt_toolkit.formatted_text import ANSI, StyleAndTextTuples 27from prompt_toolkit.formatted_text.base import OneStyleAndTextTuple 28 29from pw_presubmit.build import write_gn_args_file 30 31if TYPE_CHECKING: 32 from pw_build.project_builder import ProjectBuilder 33 from pw_build.project_builder_prefs import ProjectBuilderPrefs 34 35_LOG = logging.getLogger('pw_build.watch') 36 37 38class UnknownBuildSystem(Exception): 39 """Exception for requesting unsupported build systems.""" 40 41 42class UnknownBuildDir(Exception): 43 """Exception for an unknown build dir before command running.""" 44 45 46@dataclass 47class BuildCommand: 48 """Store details of a single build step. 49 50 Example usage: 51 52 .. code-block:: python 53 54 from pw_build.build_recipe import BuildCommand, BuildRecipe 55 56 def should_gen_gn(out: Path): 57 return not (out / 'build.ninja').is_file() 58 59 cmd1 = BuildCommand(build_dir='out', 60 command=['gn', 'gen', '{build_dir}'], 61 run_if=should_gen_gn) 62 63 cmd2 = BuildCommand(build_dir='out', 64 build_system_command='ninja', 65 build_system_extra_args=['-k', '0'], 66 targets=['default']), 67 68 Args: 69 build_dir: Output directory for this build command. This can be omitted 70 if the BuildCommand is included in the steps of a BuildRecipe. 71 build_system_command: This command should end with ``ninja``, ``make``, 72 or ``bazel``. 73 build_system_extra_args: A list of extra arguments passed to the 74 build_system_command. If running ``bazel test`` include ``test`` as 75 an extra arg here. 76 targets: Optional list of targets to build in the build_dir. 77 command: List of strings to run as a command. These are passed to 78 subprocess.run(). Any instances of the ``'{build_dir}'`` string 79 literal will be replaced at run time with the out directory. 80 run_if: A callable function to run before executing this 81 BuildCommand. The callable takes one Path arg for the build_dir. If 82 the callable returns true this command is executed. All 83 BuildCommands are run by default. 84 working_dir: Optional working directory to run build command in 85 """ 86 87 build_dir: Path | None = None 88 build_system_command: str | None = None 89 build_system_extra_args: list[str] = field(default_factory=list) 90 targets: list[str] = field(default_factory=list) 91 command: list[str] = field(default_factory=list) 92 run_if: Callable[[Path], bool] = lambda _build_dir: True 93 working_dir: Path | None = None 94 95 def __post_init__(self) -> None: 96 # Copy self._expanded_args from the command list. 97 self._expanded_args: list[str] = [] 98 if self.command: 99 self._expanded_args = self.command 100 101 def should_run(self) -> bool: 102 """Return True if this build command should be run.""" 103 if self.build_dir: 104 return self.run_if(self.build_dir) 105 return True 106 107 def _get_starting_build_system_args(self) -> list[str]: 108 """Return flags that appear immediately after the build command.""" 109 assert self.build_system_command 110 assert self.build_dir 111 return [] 112 113 def _get_build_system_args(self) -> list[str]: 114 assert self.build_system_command 115 assert self.build_dir 116 117 # Both make and ninja use -C for a build directory. 118 if self.is_make_command() or self.is_ninja_command(): 119 return ['-C', str(self.build_dir), *self.targets] 120 121 if self.is_bazel_command(): 122 # Bazel doesn't use -C for the out directory. Instead we use 123 # --symlink_prefix to save some outputs to the desired 124 # location. This is the same pattern used by pw_presubmit. 125 bazel_args = ['--symlink_prefix', str(self.build_dir / 'bazel-')] 126 if self.bazel_clean_command(): 127 # Targets are unrecognized args for bazel clean 128 return bazel_args 129 return bazel_args + [*self.targets] 130 131 raise UnknownBuildSystem( 132 f'\n\nUnknown build system command "{self.build_system_command}" ' 133 f'for build directory "{self.build_dir}".\n' 134 'Supported commands: ninja, bazel, make' 135 ) 136 137 def _resolve_expanded_args(self) -> list[str]: 138 """Replace instances of '{build_dir}' with the self.build_dir.""" 139 resolved_args = [] 140 for arg in self._expanded_args: 141 if arg == "{build_dir}": 142 if not self.build_dir: 143 raise UnknownBuildDir( 144 '\n\nUnknown "{build_dir}" value for command:\n' 145 f' {self._expanded_args}\n' 146 f'In BuildCommand: {repr(self)}\n\n' 147 'Check build_dir is set for the above BuildCommand' 148 'or included as a step to a BuildRecipe.' 149 ) 150 resolved_args.append(str(self.build_dir.resolve())) 151 else: 152 resolved_args.append(arg) 153 return resolved_args 154 155 def is_make_command(self) -> bool: 156 return ( 157 self.build_system_command is not None 158 and self.build_system_command.endswith('make') 159 ) 160 161 def is_ninja_command(self) -> bool: 162 return ( 163 self.build_system_command is not None 164 and self.build_system_command.endswith('ninja') 165 ) 166 167 def is_bazel_command(self) -> bool: 168 return self.build_system_command is not None and ( 169 self.build_system_command.endswith('bazel') 170 or self.build_system_command.endswith('bazelisk') 171 ) 172 173 def bazel_build_command(self) -> bool: 174 return ( 175 self.is_bazel_command() and 'build' in self.build_system_extra_args 176 ) 177 178 def bazel_test_command(self) -> bool: 179 return ( 180 self.is_bazel_command() and 'test' in self.build_system_extra_args 181 ) 182 183 def bazel_clean_command(self) -> bool: 184 return ( 185 self.is_bazel_command() and 'clean' in self.build_system_extra_args 186 ) 187 188 def get_args( 189 self, 190 additional_ninja_args: list[str] | None = None, 191 additional_bazel_args: list[str] | None = None, 192 additional_bazel_build_args: list[str] | None = None, 193 ) -> list[str]: 194 """Return all args required to launch this BuildCommand.""" 195 # If this is a plain command step, return self._expanded_args as-is. 196 if not self.build_system_command: 197 return self._resolve_expanded_args() 198 199 # Assmemble user-defined extra args. 200 extra_args = [] 201 extra_args.extend(self.build_system_extra_args) 202 if additional_ninja_args and self.is_ninja_command(): 203 extra_args.extend(additional_ninja_args) 204 205 if additional_bazel_build_args and self.bazel_build_command(): 206 extra_args.extend(additional_bazel_build_args) 207 208 if additional_bazel_args and self.is_bazel_command(): 209 extra_args.extend(additional_bazel_args) 210 211 build_system_target_args = self._get_build_system_args() 212 213 # Construct the build system command args. 214 command = [ 215 self.build_system_command, 216 *self._get_starting_build_system_args(), 217 *extra_args, 218 *build_system_target_args, 219 ] 220 return command 221 222 def __str__(self) -> str: 223 return ' '.join(shlex.quote(arg) for arg in self.get_args()) 224 225 226@dataclass 227class BuildRecipeStatus: 228 """Stores the status of a build recipe.""" 229 230 recipe: BuildRecipe 231 current_step: str = '' 232 percent: float = 0.0 233 error_count: int = 0 234 return_code: int | None = None 235 flag_done: bool = False 236 flag_started: bool = False 237 error_lines: dict[int, list[str]] = field(default_factory=dict) 238 239 def pending(self) -> bool: 240 return self.return_code is None 241 242 def failed(self) -> bool: 243 if self.return_code is not None: 244 return self.return_code != 0 245 return False 246 247 def append_failure_line(self, line: str) -> None: 248 lines = self.error_lines.get(self.error_count, []) 249 lines.append(line) 250 self.error_lines[self.error_count] = lines 251 252 def has_empty_ninja_errors(self) -> bool: 253 for error_lines in self.error_lines.values(): 254 # NOTE: There will be at least 2 lines for each ninja failure: 255 # - A starting 'FAILED: target' line 256 # - An ending line with this format: 257 # 'ninja: error: ... cannot make progress due to previous errors' 258 259 # If the total error line count is very short, assume it's an empty 260 # ninja error. 261 if len(error_lines) <= 3: 262 # If there is a failure in the regen step, there will be 3 error 263 # lines: The above two and one more with the regen command. 264 return True 265 # Otherwise, if the line starts with FAILED: build.ninja the failure 266 # is likely in the regen step and there will be extra cmake or gn 267 # error text that was not captured. 268 for line in error_lines: 269 if line.startswith( 270 '\033[31mFAILED: \033[0mbuild.ninja' 271 ) or line.startswith('FAILED: build.ninja'): 272 return True 273 return False 274 275 def increment_error_count(self, count: int = 1) -> None: 276 self.error_count += count 277 if self.error_count not in self.error_lines: 278 self.error_lines[self.error_count] = [] 279 280 def should_log_failures(self) -> bool: 281 return ( 282 self.recipe.project_builder is not None 283 and self.recipe.project_builder.separate_build_file_logging 284 and (not self.recipe.project_builder.send_recipe_logs_to_root) 285 ) 286 287 def log_last_failure(self) -> None: 288 """Log the last ninja error if available.""" 289 if not self.should_log_failures(): 290 return 291 292 logger = self.recipe.error_logger 293 if not logger: 294 return 295 296 _color = self.recipe.project_builder.color # type: ignore 297 298 lines = self.error_lines.get(self.error_count, []) 299 _LOG.error('') 300 _LOG.error(' ╔════════════════════════════════════') 301 _LOG.error( 302 ' ║ START %s Failure #%d:', 303 _color.cyan(self.recipe.display_name), 304 self.error_count, 305 ) 306 307 logger.error('') 308 for line in lines: 309 logger.error(line) 310 logger.error('') 311 312 _LOG.error( 313 ' ║ END %s Failure #%d', 314 _color.cyan(self.recipe.display_name), 315 self.error_count, 316 ) 317 _LOG.error(" ╚════════════════════════════════════") 318 _LOG.error('') 319 320 def log_entire_recipe_logfile(self) -> None: 321 """Log the entire build logfile if no ninja errors available.""" 322 if not self.should_log_failures(): 323 return 324 325 recipe_logfile = self.recipe.logfile 326 if not recipe_logfile: 327 return 328 329 _color = self.recipe.project_builder.color # type: ignore 330 331 logfile_path = str(recipe_logfile.resolve()) 332 333 _LOG.error('') 334 _LOG.error(' ╔════════════════════════════════════') 335 _LOG.error( 336 ' ║ %s Failure; Entire log below:', 337 _color.cyan(self.recipe.display_name), 338 ) 339 _LOG.error(' ║ %s %s', _color.yellow('START'), logfile_path) 340 341 logger = self.recipe.error_logger 342 if not logger: 343 return 344 345 logger.error('') 346 for line in recipe_logfile.read_text( 347 encoding='utf-8', errors='ignore' 348 ).splitlines(): 349 logger.error(line) 350 logger.error('') 351 352 _LOG.error(' ║ %s %s', _color.yellow('END'), logfile_path) 353 _LOG.error(" ╚════════════════════════════════════") 354 _LOG.error('') 355 356 def status_slug(self, restarting: bool = False) -> OneStyleAndTextTuple: 357 status = ('', '') 358 if not self.recipe.enabled: 359 return ('fg:ansidarkgray', 'Disabled') 360 361 waiting = False 362 if self.done: 363 if self.passed(): 364 status = ('fg:ansigreen', 'OK ') 365 elif self.failed(): 366 status = ('fg:ansired', 'FAIL ') 367 elif self.started: 368 status = ('fg:ansiyellow', 'Building') 369 else: 370 waiting = True 371 status = ('default', 'Waiting ') 372 373 # Only show Aborting if the process is building (or has failures). 374 if restarting and not waiting and not self.passed(): 375 status = ('fg:ansiyellow', 'Aborting') 376 return status 377 378 def current_step_formatted(self) -> StyleAndTextTuples: 379 formatted_text: StyleAndTextTuples = [] 380 if self.passed(): 381 return formatted_text 382 383 if self.current_step: 384 if '\x1b' in self.current_step: 385 formatted_text = ANSI(self.current_step).__pt_formatted_text__() 386 else: 387 formatted_text = [('', self.current_step)] 388 389 return formatted_text 390 391 @property 392 def done(self) -> bool: 393 return self.flag_done 394 395 @property 396 def started(self) -> bool: 397 return self.flag_started 398 399 def mark_done(self) -> None: 400 self.flag_done = True 401 402 def mark_started(self) -> None: 403 self.flag_started = True 404 405 def set_failed(self) -> None: 406 self.flag_done = True 407 self.return_code = -1 408 409 def set_passed(self) -> None: 410 self.flag_done = True 411 self.return_code = 0 412 413 def passed(self) -> bool: 414 if self.done and self.return_code is not None: 415 return self.return_code == 0 416 return False 417 418 419@dataclass 420class BuildRecipe: 421 """Dataclass to store a list of BuildCommands. 422 423 Example usage: 424 425 .. code-block:: python 426 427 from pw_build.build_recipe import BuildCommand, BuildRecipe 428 429 def should_gen_gn(out: Path) -> bool: 430 return not (out / 'build.ninja').is_file() 431 432 recipe = BuildRecipe( 433 build_dir='out', 434 title='Vanilla Ninja Build', 435 steps=[ 436 BuildCommand(command=['gn', 'gen', '{build_dir}'], 437 run_if=should_gen_gn), 438 BuildCommand(build_system_command='ninja', 439 build_system_extra_args=['-k', '0'], 440 targets=['default']), 441 ], 442 ) 443 444 Args: 445 build_dir: Output directory for this BuildRecipe. On init this out dir 446 is set for all included steps. 447 steps: List of BuildCommands to run. 448 title: Custom title. The build_dir is used if this is ommited. 449 auto_create_build_dir: Auto create the build directory and all necessary 450 parent directories before running any build commands. 451 """ 452 453 build_dir: Path 454 steps: list[BuildCommand] = field(default_factory=list) 455 title: str | None = None 456 enabled: bool = True 457 auto_create_build_dir: bool = True 458 459 def __hash__(self): 460 return hash((self.build_dir, self.title, len(self.steps))) 461 462 def __post_init__(self) -> None: 463 # Update all included steps to use this recipe's build_dir. 464 for step in self.steps: 465 if self.build_dir: 466 step.build_dir = self.build_dir 467 468 # Set logging variables 469 self._logger: logging.Logger | None = None 470 self.error_logger: logging.Logger | None = None 471 self._logfile: Path | None = None 472 self._status: BuildRecipeStatus = BuildRecipeStatus(self) 473 self.project_builder: ProjectBuilder | None = None 474 475 def toggle_enabled(self) -> None: 476 self.enabled = not self.enabled 477 478 def set_project_builder(self, project_builder) -> None: 479 self.project_builder = project_builder 480 481 def set_targets(self, new_targets: list[str]) -> None: 482 """Reset all build step targets.""" 483 for step in self.steps: 484 step.targets = new_targets 485 486 def set_logger(self, logger: logging.Logger) -> None: 487 self._logger = logger 488 489 def set_error_logger(self, logger: logging.Logger) -> None: 490 self.error_logger = logger 491 492 def set_logfile(self, log_file: Path) -> None: 493 self._logfile = log_file 494 495 def reset_status(self) -> None: 496 self._status = BuildRecipeStatus(self) 497 498 @property 499 def status(self) -> BuildRecipeStatus: 500 return self._status 501 502 @property 503 def log(self) -> logging.Logger: 504 if self._logger: 505 return self._logger 506 return logging.getLogger() 507 508 @property 509 def logfile(self) -> Path | None: 510 return self._logfile 511 512 @property 513 def display_name(self) -> str: 514 if self.title: 515 return self.title 516 return str(self.build_dir) 517 518 def targets(self) -> list[str]: 519 return list( 520 set(target for step in self.steps for target in step.targets) 521 ) 522 523 def __str__(self) -> str: 524 message = self.display_name 525 targets = self.targets() 526 if targets: 527 target_list = ' '.join(self.targets()) 528 message = f'{message} -- {target_list}' 529 return message 530 531 532def create_build_recipes(prefs: ProjectBuilderPrefs) -> list[BuildRecipe]: 533 """Create a list of BuildRecipes from ProjectBuilderPrefs.""" 534 build_recipes: list[BuildRecipe] = [] 535 536 if prefs.run_commands: 537 for command_str in prefs.run_commands: 538 build_recipes.append( 539 BuildRecipe( 540 build_dir=Path.cwd(), 541 steps=[BuildCommand(command=shlex.split(command_str))], 542 title=command_str, 543 ) 544 ) 545 546 for build_dir, targets in prefs.build_directories.items(): 547 steps: list[BuildCommand] = [] 548 build_path = Path(build_dir) 549 if not targets: 550 targets = [] 551 552 for ( 553 build_system_command, 554 build_system_extra_args, 555 ) in prefs.build_system_commands(build_dir): 556 steps.append( 557 BuildCommand( 558 build_system_command=build_system_command, 559 build_system_extra_args=build_system_extra_args, 560 targets=targets, 561 ) 562 ) 563 564 build_recipes.append( 565 BuildRecipe( 566 build_dir=build_path, 567 steps=steps, 568 ) 569 ) 570 571 return build_recipes 572 573 574def should_gn_gen(out: Path) -> bool: 575 """Returns True if the gn gen command should be run. 576 577 Returns True if ``build.ninja`` or ``args.gn`` files are missing from the 578 build directory. 579 """ 580 # gn gen only needs to run if build.ninja or args.gn files are missing. 581 expected_files = [ 582 out / 'build.ninja', 583 out / 'args.gn', 584 ] 585 return any(not gen_file.is_file() for gen_file in expected_files) 586 587 588def should_gn_gen_with_args( 589 gn_arg_dict: Mapping[str, bool | str | list | tuple] 590) -> Callable: 591 """Returns a callable which writes an args.gn file prior to checks. 592 593 Args: 594 gn_arg_dict: Dictionary of key value pairs to use as gn args. 595 596 Returns: 597 Callable which takes a single Path argument and returns a bool 598 for True if the gn gen command should be run. 599 600 The returned function will: 601 602 1. Always re-write the ``args.gn`` file. 603 2. Return True if ``build.ninja`` or ``args.gn`` files are missing. 604 """ 605 606 def _write_args_and_check(out: Path) -> bool: 607 # Always re-write the args.gn file. 608 write_gn_args_file(out / 'args.gn', **gn_arg_dict) 609 610 return should_gn_gen(out) 611 612 return _write_args_and_check 613 614 615def _should_regenerate_cmake( 616 cmake_generate_command: list[str], out: Path 617) -> bool: 618 """Save the full cmake command to a file. 619 620 Returns True if cmake files should be regenerated. 621 """ 622 _should_regenerate = True 623 cmake_command = ' '.join(cmake_generate_command) 624 cmake_command_filepath = out / 'cmake_cfg_command.txt' 625 if (out / 'build.ninja').is_file() and cmake_command_filepath.is_file(): 626 if cmake_command == cmake_command_filepath.read_text(): 627 _should_regenerate = False 628 629 if _should_regenerate: 630 out.mkdir(parents=True, exist_ok=True) 631 cmake_command_filepath.write_text(cmake_command) 632 633 return _should_regenerate 634 635 636def should_regenerate_cmake( 637 cmake_generate_command: list[str], 638) -> Callable[[Path], bool]: 639 """Return a callable to determine if cmake should be regenerated. 640 641 Args: 642 cmake_generate_command: Full list of args to run cmake. 643 644 The returned function will return True signaling CMake should be re-run if: 645 646 1. The provided CMake command does not match an existing args in the 647 ``cmake_cfg_command.txt`` file in the build dir. 648 2. ``build.ninja`` is missing or ``cmake_cfg_command.txt`` is missing. 649 650 When the function is run it will create the build directory if needed and 651 write the cmake_generate_command args to the ``cmake_cfg_command.txt`` file. 652 """ 653 return functools.partial(_should_regenerate_cmake, cmake_generate_command) 654