1# Copyright 2022 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Build a Pigweed Project. 15 16Run arbitrary commands or invoke build systems (Ninja, Bazel and make) on one or 17more build directories. 18 19Examples: 20 21.. code-block: sh 22 23 # Build the default target in out/ using ninja. 24 python -m pw_build.project_builder -C out 25 26 # Build pw_run_tests.modules in the out/cmake directory 27 python -m pw_build.project_builder -C out/cmake pw_run_tests.modules 28 29 # Build the default target in out/ and pw_apps in out/cmake 30 python -m pw_build.project_builder -C out -C out/cmake pw_apps 31 32 # Build python.tests in out/ and pw_apps in out/cmake/ 33 python -m pw_build.project_builder python.tests -C out/cmake pw_apps 34 35 # Run 'bazel build' and 'bazel test' on the target '//...' in outbazel/ 36 python -m pw_build.project_builder --run-command 'mkdir -p outbazel' 37 -C outbazel '//...' 38 --build-system-command outbazel 'bazel build' 39 --build-system-command outbazel 'bazel test' 40""" 41 42from __future__ import annotations 43 44import argparse 45import concurrent.futures 46import os 47import logging 48from pathlib import Path 49import re 50import shlex 51import sys 52import subprocess 53import time 54from typing import ( 55 Callable, 56 Generator, 57 NoReturn, 58 Sequence, 59 NamedTuple, 60) 61 62from prompt_toolkit.patch_stdout import StdoutProxy 63 64import pw_cli.env 65import pw_cli.log 66 67from pw_build.build_recipe import BuildRecipe, create_build_recipes 68from pw_build.project_builder_argparse import add_project_builder_arguments 69from pw_build.project_builder_context import get_project_builder_context 70from pw_build.project_builder_prefs import ProjectBuilderPrefs 71 72_COLOR = pw_cli.color.colors() 73_LOG = logging.getLogger('pw_build') 74 75BUILDER_CONTEXT = get_project_builder_context() 76 77PASS_MESSAGE = """ 78 ██████╗ █████╗ ███████╗███████╗██╗ 79 ██╔══██╗██╔══██╗██╔════╝██╔════╝██║ 80 ██████╔╝███████║███████╗███████╗██║ 81 ██╔═══╝ ██╔══██║╚════██║╚════██║╚═╝ 82 ██║ ██║ ██║███████║███████║██╗ 83 ╚═╝ ╚═╝ ╚═╝╚══════╝╚══════╝╚═╝ 84""" 85 86# Pick a visually-distinct font from "PASS" to ensure that readers can't 87# possibly mistake the difference between the two states. 88FAIL_MESSAGE = """ 89 ▄██████▒░▄▄▄ ██▓ ░██▓ 90 ▓█▓ ░▒████▄ ▓██▒ ░▓██▒ 91 ▒████▒ ░▒█▀ ▀█▄ ▒██▒ ▒██░ 92 ░▓█▒ ░░██▄▄▄▄██ ░██░ ▒██░ 93 ░▒█░ ▓█ ▓██▒░██░░ ████████▒ 94 ▒█░ ▒▒ ▓▒█░░▓ ░ ▒░▓ ░ 95 ░▒ ▒ ▒▒ ░ ▒ ░░ ░ ▒ ░ 96 ░ ░ ░ ▒ ▒ ░ ░ ░ 97 ░ ░ ░ ░ ░ 98""" 99 100 101class ProjectBuilderCharset(NamedTuple): 102 slug_ok: str 103 slug_fail: str 104 slug_building: str 105 106 107ASCII_CHARSET = ProjectBuilderCharset( 108 _COLOR.green('OK '), 109 _COLOR.red('FAIL'), 110 _COLOR.yellow('... '), 111) 112EMOJI_CHARSET = ProjectBuilderCharset('✔️ ', '❌', '⏱️ ') 113 114 115def _exit(*args) -> NoReturn: 116 _LOG.critical(*args) 117 sys.exit(1) 118 119 120def _exit_due_to_interrupt() -> None: 121 """Abort function called when not using progress bars.""" 122 # To keep the log lines aligned with each other in the presence of 123 # a '^C' from the keyboard interrupt, add a newline before the log. 124 print() 125 _LOG.info('Got Ctrl-C; exiting...') 126 BUILDER_CONTEXT.ctrl_c_interrupt() 127 128 129_NINJA_BUILD_STEP = re.compile( 130 # Start of line 131 r'^' 132 # Step count: [1234/5678] 133 r'\[(?P<step>[0-9]+)/(?P<total_steps>[0-9]+)\]' 134 # whitespace 135 r' *' 136 # Build step text 137 r'(?P<action>.*)$' 138) 139 140_BAZEL_BUILD_STEP = re.compile( 141 # Start of line 142 r'^' 143 # Optional starting green color 144 r'(?:\x1b\[32m)?' 145 # Step count: [1,234 / 5,678] 146 r'\[(?P<step>[0-9,]+) */ *(?P<total_steps>[0-9,]+)\]' 147 # Optional ending clear color and space 148 r'(?:\x1b\[0m)? *' 149 # Build step text 150 r'(?P<action>.*)$' 151) 152 153_NINJA_FAILURE = re.compile( 154 # Start of line 155 r'^' 156 # Optional red color 157 r'(?:\x1b\[31m)?' 158 r'FAILED:' 159 r' ' 160 # Optional color reset 161 r'(?:\x1b\[0m)?' 162) 163 164_BAZEL_FAILURE = re.compile( 165 # Start of line 166 r'^' 167 # Optional red color 168 r'(?:\x1b\[31m)?' 169 # Optional bold color 170 r'(?:\x1b\[1m)?' 171 r'FAIL:' 172 # Optional color reset 173 r'(?:\x1b\[0m)?' 174 # whitespace 175 r' *' 176 r'.*bazel-out.*' 177) 178 179_BAZEL_ELAPSED_TIME = re.compile( 180 # Start of line 181 r'^' 182 # Optional green color 183 r'(?:\x1b\[32m)?' 184 r'INFO:' 185 # single space 186 r' ' 187 # Optional color reset 188 r'(?:\x1b\[0m)?' 189 r'Elapsed time:' 190) 191 192 193def execute_command_no_logging( 194 command: list, 195 env: dict, 196 recipe: BuildRecipe, 197 working_dir: Path | None = None, 198 # pylint: disable=unused-argument 199 logger: logging.Logger = _LOG, 200 line_processed_callback: Callable[[BuildRecipe], None] | None = None, 201 # pylint: enable=unused-argument 202) -> bool: 203 print() 204 proc = subprocess.Popen(command, env=env, cwd=working_dir, errors='replace') 205 BUILDER_CONTEXT.register_process(recipe, proc) 206 returncode = None 207 while returncode is None: 208 if BUILDER_CONTEXT.build_stopping(): 209 try: 210 proc.terminate() 211 except ProcessLookupError: 212 # Process already stopped. 213 pass 214 returncode = proc.poll() 215 time.sleep(0.05) 216 print() 217 recipe.status.return_code = returncode 218 219 return proc.returncode == 0 220 221 222def execute_command_with_logging( 223 command: list, 224 env: dict, 225 recipe: BuildRecipe, 226 working_dir: Path | None = None, 227 logger: logging.Logger = _LOG, 228 line_processed_callback: Callable[[BuildRecipe], None] | None = None, 229) -> bool: 230 """Run a command in a subprocess and log all output.""" 231 current_stdout = '' 232 returncode = None 233 234 starting_failure_regex = _NINJA_FAILURE 235 ending_failure_regex = _NINJA_BUILD_STEP 236 build_step_regex = _NINJA_BUILD_STEP 237 238 if command[0].endswith('ninja'): 239 build_step_regex = _NINJA_BUILD_STEP 240 starting_failure_regex = _NINJA_FAILURE 241 ending_failure_regex = _NINJA_BUILD_STEP 242 elif command[0].endswith('bazel'): 243 build_step_regex = _BAZEL_BUILD_STEP 244 starting_failure_regex = _BAZEL_FAILURE 245 ending_failure_regex = _BAZEL_ELAPSED_TIME 246 247 with subprocess.Popen( 248 command, 249 env=env, 250 cwd=working_dir, 251 stdout=subprocess.PIPE, 252 stderr=subprocess.STDOUT, 253 errors='replace', 254 ) as proc: 255 BUILDER_CONTEXT.register_process(recipe, proc) 256 should_log_build_steps = BUILDER_CONTEXT.log_build_steps 257 # Empty line at the start. 258 logger.info('') 259 260 failure_line = False 261 while returncode is None: 262 output = '' 263 error_output = '' 264 265 if proc.stdout: 266 output = proc.stdout.readline() 267 current_stdout += output 268 if proc.stderr: 269 error_output += proc.stderr.readline() 270 current_stdout += error_output 271 272 if not output and not error_output: 273 returncode = proc.poll() 274 continue 275 276 line_match_result = build_step_regex.match(output) 277 if line_match_result: 278 if failure_line and not BUILDER_CONTEXT.build_stopping(): 279 recipe.status.log_last_failure() 280 failure_line = False 281 matches = line_match_result.groupdict() 282 recipe.status.current_step = line_match_result.group(0) 283 recipe.status.percent = 0.0 284 # Remove commas from step and total_steps strings 285 step = int(matches.get('step', '0').replace(',', '')) 286 total_steps = int( 287 matches.get('total_steps', '1').replace(',', '') 288 ) 289 if total_steps > 0: 290 recipe.status.percent = float(step / total_steps) 291 292 logger_method = logger.info 293 if starting_failure_regex.match(output): 294 logger_method = logger.error 295 if failure_line and not BUILDER_CONTEXT.build_stopping(): 296 recipe.status.log_last_failure() 297 recipe.status.increment_error_count() 298 failure_line = True 299 elif ending_failure_regex.match(output): 300 if failure_line and not BUILDER_CONTEXT.build_stopping(): 301 recipe.status.log_last_failure() 302 failure_line = False 303 304 # Mypy output mixes character encoding in color coded output 305 # and uses the 'sgr0' (or exit_attribute_mode) capability from the 306 # host machine's terminfo database. 307 # 308 # This can result in this sequence ending up in STDOUT as 309 # b'\x1b(B\x1b[m'. (B tells terminals to interpret text as 310 # USASCII encoding but will appear in prompt_toolkit as a B 311 # character. 312 # 313 # The following replace calls will strip out those 314 # sequences. 315 stripped_output = output.replace('\x1b(B', '').strip() 316 317 # If this isn't a build step. 318 if not line_match_result or ( 319 # Or if this is a build step and logging build steps is 320 # requested: 321 line_match_result 322 and should_log_build_steps 323 ): 324 # Log this line. 325 logger_method(stripped_output) 326 recipe.status.current_step = stripped_output 327 328 if failure_line: 329 recipe.status.append_failure_line(stripped_output) 330 331 BUILDER_CONTEXT.redraw_progress() 332 333 if line_processed_callback: 334 line_processed_callback(recipe) 335 336 if BUILDER_CONTEXT.build_stopping(): 337 try: 338 proc.terminate() 339 except ProcessLookupError: 340 # Process already stopped. 341 pass 342 343 recipe.status.return_code = returncode 344 345 # Log the last failure if not done already 346 if failure_line and not BUILDER_CONTEXT.build_stopping(): 347 recipe.status.log_last_failure() 348 349 # Empty line at the end. 350 logger.info('') 351 352 return returncode == 0 353 354 355def log_build_recipe_start( 356 index_message: str, 357 project_builder: ProjectBuilder, 358 cfg: BuildRecipe, 359 logger: logging.Logger = _LOG, 360) -> None: 361 """Log recipe start and truncate the build logfile.""" 362 if project_builder.separate_build_file_logging and cfg.logfile: 363 # Truncate the file 364 with open(cfg.logfile, 'w'): 365 pass 366 367 BUILDER_CONTEXT.mark_progress_started(cfg) 368 369 build_start_msg = [ 370 index_message, 371 project_builder.color.cyan('Starting ==>'), 372 project_builder.color.blue('Recipe:'), 373 str(cfg.display_name), 374 project_builder.color.blue('Targets:'), 375 str(' '.join(cfg.targets())), 376 ] 377 378 if cfg.logfile: 379 build_start_msg.extend( 380 [ 381 project_builder.color.blue('Logfile:'), 382 str(cfg.logfile.resolve()), 383 ] 384 ) 385 build_start_str = ' '.join(build_start_msg) 386 387 # Log start to the root log if recipe logs are not sent. 388 if not project_builder.send_recipe_logs_to_root: 389 logger.info(build_start_str) 390 if cfg.logfile: 391 cfg.log.info(build_start_str) 392 393 394def log_build_recipe_finish( 395 index_message: str, 396 project_builder: ProjectBuilder, 397 cfg: BuildRecipe, 398 logger: logging.Logger = _LOG, 399) -> None: 400 """Log recipe finish and any build errors.""" 401 402 BUILDER_CONTEXT.mark_progress_done(cfg) 403 404 if BUILDER_CONTEXT.interrupted(): 405 level = logging.WARNING 406 tag = project_builder.color.yellow('(ABORT)') 407 elif cfg.status.failed(): 408 level = logging.ERROR 409 tag = project_builder.color.red('(FAIL)') 410 else: 411 level = logging.INFO 412 tag = project_builder.color.green('(OK)') 413 414 build_finish_msg = [ 415 level, 416 '%s %s %s %s %s', 417 index_message, 418 project_builder.color.cyan('Finished ==>'), 419 project_builder.color.blue('Recipe:'), 420 cfg.display_name, 421 tag, 422 ] 423 424 # Log finish to the root log if recipe logs are not sent. 425 if not project_builder.send_recipe_logs_to_root: 426 logger.log(*build_finish_msg) 427 if cfg.logfile: 428 cfg.log.log(*build_finish_msg) 429 430 if ( 431 not BUILDER_CONTEXT.build_stopping() 432 and cfg.status.failed() 433 and (cfg.status.error_count == 0 or cfg.status.has_empty_ninja_errors()) 434 ): 435 cfg.status.log_entire_recipe_logfile() 436 437 438class MissingGlobalLogfile(Exception): 439 """Exception raised if a global logfile is not specifed.""" 440 441 442class DispatchingFormatter(logging.Formatter): 443 """Dispatch log formatting based on the logger name.""" 444 445 def __init__(self, formatters, default_formatter): 446 self._formatters = formatters 447 self._default_formatter = default_formatter 448 super().__init__() 449 450 def format(self, record): 451 logger = logging.getLogger(record.name) 452 formatter = self._formatters.get(logger.name, self._default_formatter) 453 return formatter.format(record) 454 455 456class ProjectBuilder: # pylint: disable=too-many-instance-attributes 457 """Pigweed Project Builder 458 459 Controls how build recipes are executed and logged. 460 461 Example usage: 462 463 .. code-block:: python 464 465 import logging 466 from pathlib import Path 467 468 from pw_build.build_recipe import BuildCommand, BuildRecipe 469 from pw_build.project_builder import ProjectBuilder 470 471 def should_gen_gn(out: Path) -> bool: 472 return not (out / 'build.ninja').is_file() 473 474 recipe = BuildRecipe( 475 build_dir='out', 476 title='Vanilla Ninja Build', 477 steps=[ 478 BuildCommand(command=['gn', 'gen', '{build_dir}'], 479 run_if=should_gen_gn), 480 BuildCommand(build_system_command='ninja', 481 build_system_extra_args=['-k', '0'], 482 targets=['default']), 483 ], 484 ) 485 486 project_builder = ProjectBuilder( 487 build_recipes=[recipe1, ...] 488 banners=True, 489 log_level=logging.INFO 490 separate_build_file_logging=True, 491 root_logger=logging.getLogger(), 492 root_logfile=Path('build_log.txt'), 493 ) 494 495 Args: 496 build_recipes: List of build recipes. 497 jobs: The number of jobs bazel, make, and ninja should use by passing 498 ``-j`` to each. 499 keep_going: If True keep going flags are passed to bazel and ninja with 500 the ``-k`` option. 501 banners: Print the project banner at the start of each build. 502 allow_progress_bars: If False progress bar output will be disabled. 503 log_build_steps: If True all build step lines will be logged to the 504 screen and logfiles. Default: False. 505 colors: Print ANSI colors to stdout and logfiles 506 log_level: Optional log_level, defaults to logging.INFO. 507 root_logfile: Optional root logfile. 508 separate_build_file_logging: If True separate logfiles will be created 509 per build recipe. The location of each file depends on if a 510 ``root_logfile`` is provided. If a root logfile is used each build 511 recipe logfile will be created in the same location. If no 512 root_logfile is specified the per build log files are placed in each 513 build dir as ``log.txt`` 514 send_recipe_logs_to_root: If True will send all build recipie output to 515 the root logger. This only makes sense to use if the builds are run 516 in serial. 517 use_verbatim_error_log_formatting: Use a blank log format when printing 518 errors from sub builds to the root logger. 519 source_path: Path to the root of the source files. Defaults to the 520 current working directory. If running under bazel this will be set 521 to the $BUILD_WORKSPACE_DIRECTORY environment variable. Otherwise 522 $PW_PROJECT_ROOT will be used. 523 """ 524 525 def __init__( 526 # pylint: disable=too-many-arguments,too-many-locals 527 self, 528 build_recipes: Sequence[BuildRecipe], 529 jobs: int | None = None, 530 banners: bool = True, 531 keep_going: bool = False, 532 abort_callback: Callable = _exit, 533 execute_command: Callable[ 534 [ 535 list, 536 dict, 537 BuildRecipe, 538 Path | None, 539 logging.Logger, 540 Callable | None, 541 ], 542 bool, 543 ] = execute_command_no_logging, 544 charset: ProjectBuilderCharset = ASCII_CHARSET, 545 colors: bool = True, 546 separate_build_file_logging: bool = False, 547 send_recipe_logs_to_root: bool = False, 548 root_logger: logging.Logger = _LOG, 549 root_logfile: Path | None = None, 550 log_level: int = logging.INFO, 551 allow_progress_bars: bool = True, 552 use_verbatim_error_log_formatting: bool = False, 553 log_build_steps: bool = False, 554 source_path: Path | None = None, 555 ): 556 self.charset: ProjectBuilderCharset = charset 557 self.abort_callback = abort_callback 558 # Function used to run subprocesses 559 self.execute_command = execute_command 560 self.banners = banners 561 self.build_recipes = build_recipes 562 self.max_name_width = max( 563 [len(str(step.display_name)) for step in self.build_recipes] 564 ) 565 # Set project_builder reference in each recipe. 566 for recipe in self.build_recipes: 567 recipe.set_project_builder(self) 568 569 # Save build system args 570 self.extra_ninja_args: list[str] = [] 571 self.extra_bazel_args: list[str] = [] 572 self.extra_bazel_build_args: list[str] = [] 573 574 # Handle jobs and keep going flags. 575 if jobs: 576 job_args = ['-j', f'{jobs}'] 577 self.extra_ninja_args.extend(job_args) 578 self.extra_bazel_build_args.extend(job_args) 579 if keep_going: 580 self.extra_ninja_args.extend(['-k', '0']) 581 self.extra_bazel_build_args.extend(['-k']) 582 583 self.colors = colors 584 # Reference to pw_cli.color, will return colored text if colors are 585 # enabled. 586 self.color = pw_cli.color.colors(colors) 587 588 # Pass color setting to bazel 589 if colors: 590 self.extra_bazel_args.append('--color=yes') 591 else: 592 self.extra_bazel_args.append('--color=no') 593 594 # Progress bar enable/disable flag 595 self.allow_progress_bars = allow_progress_bars 596 597 # Disable progress bars if the terminal is not a tty or the 598 # $TERM env var is set accordingly. 599 term = os.environ.get('TERM', '') 600 # inclusive-language: disable 601 if not sys.stdout.isatty() or term.lower() in ['dumb', 'unknown']: 602 self.allow_progress_bars = False 603 # inclusive-language: enable 604 605 self.log_build_steps = log_build_steps 606 self.stdout_proxy: StdoutProxy | None = None 607 608 # Logger configuration 609 self.root_logger = root_logger 610 self.default_logfile = root_logfile 611 self.default_log_level = log_level 612 # Create separate logs per build 613 self.separate_build_file_logging = separate_build_file_logging 614 # Propagate logs to the root looger 615 self.send_recipe_logs_to_root = send_recipe_logs_to_root 616 617 # Setup the error logger 618 self.use_verbatim_error_log_formatting = ( 619 use_verbatim_error_log_formatting 620 ) 621 self.error_logger = logging.getLogger(f'{root_logger.name}.errors') 622 self.error_logger.setLevel(log_level) 623 self.error_logger.propagate = True 624 for recipe in self.build_recipes: 625 recipe.set_error_logger(self.error_logger) 626 627 # Copy of the standard Pigweed style log formatter, used by default if 628 # no formatter exists on the root logger. 629 timestamp_fmt = self.color.black_on_white('%(asctime)s') + ' ' 630 self.default_log_formatter = logging.Formatter( 631 timestamp_fmt + '%(levelname)s %(message)s', '%Y%m%d %H:%M:%S' 632 ) 633 634 # Empty log formatter (optionally used for error reporting) 635 self.blank_log_formatter = logging.Formatter('%(message)s') 636 637 # Setup the default log handler and inherit user defined formatting on 638 # the root_logger. 639 self.apply_root_log_formatting() 640 641 # Create a root logfile to save what is normally logged to stdout. 642 if root_logfile: 643 # Execute subprocesses and capture logs 644 self.execute_command = execute_command_with_logging 645 646 root_logfile.parent.mkdir(parents=True, exist_ok=True) 647 648 build_log_filehandler = logging.FileHandler( 649 root_logfile, 650 encoding='utf-8', 651 # Truncate the file 652 mode='w', 653 ) 654 build_log_filehandler.setLevel(log_level) 655 build_log_filehandler.setFormatter(self.dispatching_log_formatter) 656 root_logger.addHandler(build_log_filehandler) 657 658 # Set each recipe to use the root logger by default. 659 for recipe in self.build_recipes: 660 recipe.set_logger(root_logger) 661 662 # Create separate logfiles per build 663 if separate_build_file_logging: 664 self._create_per_build_logfiles() 665 666 self.source_path = source_path 667 668 # Determine the source root path. 669 if not self.source_path: 670 self.source_path = pw_cli.env.project_root() 671 672 # If source_path was set change to that directory before building. 673 if self.source_path: 674 os.chdir(self.source_path) 675 676 def _create_per_build_logfiles(self) -> None: 677 """Create separate log files per build. 678 679 If a root logfile is used, create per build log files in the same 680 location. If no root logfile is specified create the per build log files 681 in the build dir as ``log.txt`` 682 """ 683 self.execute_command = execute_command_with_logging 684 685 for recipe in self.build_recipes: 686 sub_logger_name = recipe.display_name.replace('.', '_') 687 new_logger = logging.getLogger( 688 f'{self.root_logger.name}.{sub_logger_name}' 689 ) 690 new_logger.setLevel(self.default_log_level) 691 new_logger.propagate = self.send_recipe_logs_to_root 692 693 new_logfile_dir = recipe.build_dir 694 new_logfile_name = Path('log.txt') 695 new_logfile_postfix = '' 696 if self.default_logfile: 697 new_logfile_dir = self.default_logfile.parent 698 new_logfile_name = self.default_logfile 699 # Replace spaces and forward slash with undescores. 700 display_name = recipe.display_name 701 display_name = display_name.replace(' ', '_') 702 display_name = display_name.replace('/', '_') 703 new_logfile_postfix = '_' + display_name 704 705 new_logfile = new_logfile_dir / ( 706 new_logfile_name.stem 707 + new_logfile_postfix 708 + new_logfile_name.suffix 709 ) 710 711 new_logfile_dir.mkdir(parents=True, exist_ok=True) 712 new_log_filehandler = logging.FileHandler( 713 new_logfile, 714 encoding='utf-8', 715 # Truncate the file 716 mode='w', 717 ) 718 new_log_filehandler.setLevel(self.default_log_level) 719 new_log_filehandler.setFormatter(self.dispatching_log_formatter) 720 new_logger.addHandler(new_log_filehandler) 721 722 recipe.set_logger(new_logger) 723 recipe.set_logfile(new_logfile) 724 725 def apply_root_log_formatting(self) -> None: 726 """Inherit user defined formatting from the root_logger.""" 727 # Use the the existing root logger formatter if one exists. 728 for handler in logging.getLogger().handlers: 729 if handler.formatter: 730 self.default_log_formatter = handler.formatter 731 break 732 733 formatter_mapping = { 734 self.root_logger.name: self.default_log_formatter, 735 } 736 if self.use_verbatim_error_log_formatting: 737 formatter_mapping[self.error_logger.name] = self.blank_log_formatter 738 739 self.dispatching_log_formatter = DispatchingFormatter( 740 formatter_mapping, 741 self.default_log_formatter, 742 ) 743 744 def should_use_progress_bars(self) -> bool: 745 if not self.allow_progress_bars: 746 return False 747 if self.separate_build_file_logging or self.default_logfile: 748 return True 749 return False 750 751 def use_stdout_proxy(self) -> None: 752 """Setup StdoutProxy for progress bars.""" 753 754 self.stdout_proxy = StdoutProxy(raw=True) 755 root_logger = logging.getLogger() 756 handlers = root_logger.handlers + self.error_logger.handlers 757 758 for handler in handlers: 759 # Must use type() check here since this returns True: 760 # isinstance(logging.FileHandler, logging.StreamHandler) 761 # pylint: disable=unidiomatic-typecheck 762 if type(handler) == logging.StreamHandler: 763 handler.setStream(self.stdout_proxy) # type: ignore 764 handler.setFormatter(self.dispatching_log_formatter) 765 # pylint: enable=unidiomatic-typecheck 766 767 def flush_log_handlers(self) -> None: 768 root_logger = logging.getLogger() 769 handlers = root_logger.handlers + self.error_logger.handlers 770 for cfg in self: 771 handlers.extend(cfg.log.handlers) 772 for handler in handlers: 773 handler.flush() 774 if self.stdout_proxy: 775 self.stdout_proxy.flush() 776 self.stdout_proxy.close() 777 778 def __len__(self) -> int: 779 return len(self.build_recipes) 780 781 def __getitem__(self, index: int) -> BuildRecipe: 782 return self.build_recipes[index] 783 784 def __iter__(self) -> Generator[BuildRecipe, None, None]: 785 return (build_recipe for build_recipe in self.build_recipes) 786 787 def run_build( 788 self, 789 cfg: BuildRecipe, 790 env: dict, 791 index_message: str | None = '', 792 ) -> bool: 793 """Run a single build config.""" 794 if BUILDER_CONTEXT.build_stopping(): 795 return False 796 797 if self.colors: 798 # Force colors in Pigweed subcommands run through the watcher. 799 env['PW_USE_COLOR'] = '1' 800 # Force Ninja to output ANSI colors 801 env['CLICOLOR_FORCE'] = '1' 802 803 build_succeeded = False 804 cfg.reset_status() 805 cfg.status.mark_started() 806 807 if cfg.auto_create_build_dir: 808 cfg.build_dir.mkdir(parents=True, exist_ok=True) 809 810 for command_step in cfg.steps: 811 command_args = command_step.get_args( 812 additional_ninja_args=self.extra_ninja_args, 813 additional_bazel_args=self.extra_bazel_args, 814 additional_bazel_build_args=self.extra_bazel_build_args, 815 ) 816 817 quoted_command_args = ' '.join( 818 shlex.quote(arg) for arg in command_args 819 ) 820 build_succeeded = True 821 if command_step.should_run(): 822 cfg.log.info( 823 '%s %s %s', 824 index_message, 825 self.color.blue('Run ==>'), 826 quoted_command_args, 827 ) 828 829 # Verify that the build output directories exist. 830 if ( 831 command_step.build_system_command is not None 832 and cfg.build_dir 833 and (not cfg.build_dir.is_dir()) 834 ): 835 self.abort_callback( 836 'Build directory does not exist: %s', cfg.build_dir 837 ) 838 839 build_succeeded = self.execute_command( 840 command_args, 841 env, 842 cfg, 843 command_step.working_dir, 844 cfg.log, 845 None, 846 ) 847 else: 848 cfg.log.info( 849 '%s %s %s', 850 index_message, 851 self.color.yellow('Skipped ==>'), 852 quoted_command_args, 853 ) 854 855 BUILDER_CONTEXT.mark_progress_step_complete(cfg) 856 # Don't run further steps if a command fails. 857 if not build_succeeded: 858 break 859 860 # If all steps were skipped the return code will not be set. Force 861 # status to passed in this case. 862 if build_succeeded and not cfg.status.passed(): 863 cfg.status.set_passed() 864 865 cfg.status.mark_done() 866 867 return build_succeeded 868 869 def print_pass_fail_banner( 870 self, 871 cancelled: bool = False, 872 logger: logging.Logger = _LOG, 873 ) -> None: 874 # Check conditions where banners should not be shown: 875 # Banner flag disabled. 876 if not self.banners: 877 return 878 # If restarting or interrupted. 879 if BUILDER_CONTEXT.interrupted(): 880 if BUILDER_CONTEXT.ctrl_c_pressed: 881 _LOG.info( 882 self.color.yellow('Exited due to keyboard interrupt.') 883 ) 884 return 885 # If any build is still pending. 886 if any(recipe.status.pending() for recipe in self): 887 return 888 889 # Show a large color banner for the overall result. 890 if all(recipe.status.passed() for recipe in self) and not cancelled: 891 for line in PASS_MESSAGE.splitlines(): 892 logger.info(self.color.green(line)) 893 else: 894 for line in FAIL_MESSAGE.splitlines(): 895 logger.info(self.color.red(line)) 896 897 def print_build_summary( 898 self, 899 cancelled: bool = False, 900 logger: logging.Logger = _LOG, 901 ) -> None: 902 """Print build status summary table.""" 903 904 build_descriptions = [] 905 build_status = [] 906 907 for cfg in self: 908 description = [str(cfg.display_name).ljust(self.max_name_width)] 909 description.append(' '.join(cfg.targets())) 910 build_descriptions.append(' '.join(description)) 911 912 if cfg.status.passed(): 913 build_status.append(self.charset.slug_ok) 914 elif cfg.status.failed(): 915 build_status.append(self.charset.slug_fail) 916 else: 917 build_status.append(self.charset.slug_building) 918 919 if not cancelled: 920 logger.info(' ╔════════════════════════════════════') 921 logger.info(' ║') 922 923 for slug, cmd in zip(build_status, build_descriptions): 924 logger.info(' ║ %s %s', slug, cmd) 925 926 logger.info(' ║') 927 logger.info(" ╚════════════════════════════════════") 928 else: 929 # Build was interrupted. 930 logger.info('') 931 logger.info(' ╔════════════════════════════════════') 932 logger.info(' ║') 933 logger.info(' ║ %s- interrupted', self.charset.slug_fail) 934 logger.info(' ║') 935 logger.info(" ╚════════════════════════════════════") 936 937 938def run_recipe( 939 index: int, project_builder: ProjectBuilder, cfg: BuildRecipe, env 940) -> bool: 941 if BUILDER_CONTEXT.interrupted(): 942 return False 943 if not cfg.enabled: 944 return False 945 946 num_builds = len(project_builder) 947 index_message = f'[{index}/{num_builds}]' 948 949 result = False 950 951 log_build_recipe_start(index_message, project_builder, cfg) 952 953 result = project_builder.run_build(cfg, env, index_message=index_message) 954 955 log_build_recipe_finish(index_message, project_builder, cfg) 956 957 return result 958 959 960def run_builds(project_builder: ProjectBuilder, workers: int = 1) -> int: 961 """Execute all build recipe steps. 962 963 Args: 964 project_builder: A ProjectBuilder instance 965 workers: The number of build recipes that should be run in 966 parallel. Defaults to 1 or no parallel execution. 967 968 Returns: 969 1 for a failed build, 0 for success. 970 """ 971 num_builds = len(project_builder) 972 _LOG.info('Starting build with %d directories', num_builds) 973 if project_builder.default_logfile: 974 _LOG.info( 975 '%s %s', 976 project_builder.color.blue('Root logfile:'), 977 project_builder.default_logfile.resolve(), 978 ) 979 980 env = os.environ.copy() 981 982 # Print status before starting 983 if not project_builder.should_use_progress_bars(): 984 project_builder.print_build_summary() 985 project_builder.print_pass_fail_banner() 986 987 if workers > 1 and not project_builder.separate_build_file_logging: 988 _LOG.warning( 989 project_builder.color.yellow( 990 'Running in parallel without --separate-logfiles; All build ' 991 'output will be interleaved.' 992 ) 993 ) 994 995 BUILDER_CONTEXT.set_project_builder(project_builder) 996 BUILDER_CONTEXT.set_building() 997 998 def _cleanup() -> None: 999 if not project_builder.should_use_progress_bars(): 1000 project_builder.print_build_summary() 1001 project_builder.print_pass_fail_banner() 1002 project_builder.flush_log_handlers() 1003 BUILDER_CONTEXT.set_idle() 1004 BUILDER_CONTEXT.exit_progress() 1005 1006 if workers == 1: 1007 # TODO(tonymd): Try to remove this special case. Using 1008 # ThreadPoolExecutor when running in serial (workers==1) currently 1009 # breaks Ctrl-C handling. Build processes keep running. 1010 try: 1011 if project_builder.should_use_progress_bars(): 1012 BUILDER_CONTEXT.add_progress_bars() 1013 for i, cfg in enumerate(project_builder, start=1): 1014 run_recipe(i, project_builder, cfg, env) 1015 # Ctrl-C on Unix generates KeyboardInterrupt 1016 # Ctrl-Z on Windows generates EOFError 1017 except (KeyboardInterrupt, EOFError): 1018 _exit_due_to_interrupt() 1019 finally: 1020 _cleanup() 1021 1022 else: 1023 with concurrent.futures.ThreadPoolExecutor( 1024 max_workers=workers 1025 ) as executor: 1026 futures = [] 1027 for i, cfg in enumerate(project_builder, start=1): 1028 futures.append( 1029 executor.submit(run_recipe, i, project_builder, cfg, env) 1030 ) 1031 1032 try: 1033 if project_builder.should_use_progress_bars(): 1034 BUILDER_CONTEXT.add_progress_bars() 1035 for future in concurrent.futures.as_completed(futures): 1036 future.result() 1037 # Ctrl-C on Unix generates KeyboardInterrupt 1038 # Ctrl-Z on Windows generates EOFError 1039 except (KeyboardInterrupt, EOFError): 1040 _exit_due_to_interrupt() 1041 finally: 1042 _cleanup() 1043 1044 project_builder.flush_log_handlers() 1045 return BUILDER_CONTEXT.exit_code() 1046 1047 1048def main() -> int: 1049 """Build a Pigweed Project.""" 1050 parser = argparse.ArgumentParser( 1051 description=__doc__, 1052 formatter_class=argparse.RawDescriptionHelpFormatter, 1053 ) 1054 parser = add_project_builder_arguments(parser) 1055 args = parser.parse_args() 1056 1057 pw_env = pw_cli.env.pigweed_environment() 1058 if pw_env.PW_EMOJI: 1059 charset = EMOJI_CHARSET 1060 else: 1061 charset = ASCII_CHARSET 1062 1063 prefs = ProjectBuilderPrefs( 1064 load_argparse_arguments=add_project_builder_arguments 1065 ) 1066 prefs.apply_command_line_args(args) 1067 build_recipes = create_build_recipes(prefs) 1068 1069 log_level = logging.DEBUG if args.debug_logging else logging.INFO 1070 1071 pw_cli.log.install( 1072 level=log_level, 1073 use_color=args.colors, 1074 hide_timestamp=False, 1075 ) 1076 1077 project_builder = ProjectBuilder( 1078 build_recipes=build_recipes, 1079 jobs=args.jobs, 1080 banners=args.banners, 1081 keep_going=args.keep_going, 1082 colors=args.colors, 1083 charset=charset, 1084 separate_build_file_logging=args.separate_logfiles, 1085 root_logfile=args.logfile, 1086 root_logger=_LOG, 1087 log_level=log_level, 1088 ) 1089 1090 if project_builder.should_use_progress_bars(): 1091 project_builder.use_stdout_proxy() 1092 1093 workers = 1 1094 if args.parallel: 1095 # If parallel is requested and parallel_workers is set to 0 run all 1096 # recipes in parallel. That is, use the number of recipes as the worker 1097 # count. 1098 if args.parallel_workers == 0: 1099 workers = len(project_builder) 1100 else: 1101 workers = args.parallel_workers 1102 1103 return run_builds(project_builder, workers) 1104 1105 1106if __name__ == '__main__': 1107 sys.exit(main()) 1108