1# Copyright 2020 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""Functions for building code during presubmit checks.""" 15 16import base64 17import contextlib 18from dataclasses import dataclass 19import io 20import itertools 21import json 22import logging 23import os 24import posixpath 25from pathlib import Path 26import re 27import subprocess 28from shutil import which 29import sys 30import tarfile 31from typing import ( 32 Any, 33 Callable, 34 Collection, 35 Container, 36 ContextManager, 37 Iterable, 38 Iterator, 39 Mapping, 40 Sequence, 41 Set, 42) 43 44import pw_cli.color 45from pw_cli.plural import plural 46from pw_cli.file_filter import FileFilter 47from pw_presubmit.presubmit import ( 48 call, 49 Check, 50 filter_paths, 51 install_package, 52 PresubmitResult, 53 SubStep, 54) 55from pw_presubmit.presubmit_context import ( 56 PresubmitContext, 57 PresubmitFailure, 58) 59from pw_presubmit import ( 60 bazel_parser, 61 format_code, 62 ninja_parser, 63) 64from pw_presubmit.tools import ( 65 log_run, 66 format_command, 67) 68 69_LOG = logging.getLogger(__name__) 70 71 72BAZEL_EXECUTABLE = 'bazel' 73 74 75def bazel( 76 ctx: PresubmitContext, 77 cmd: str, 78 *args: str, 79 strict_module_lockfile: bool = False, 80 use_remote_cache: bool = False, 81 stdout: io.TextIOWrapper | None = None, 82 **kwargs, 83) -> None: 84 """Invokes Bazel with some common flags set. 85 86 Intended for use with bazel build and test. May not work with others. 87 """ 88 89 num_jobs: list[str] = [] 90 if ctx.num_jobs is not None: 91 num_jobs.extend(('--jobs', str(ctx.num_jobs))) 92 93 keep_going: list[str] = [] 94 if ctx.continue_after_build_error: 95 keep_going.append('--keep_going') 96 97 strict_lockfile: list[str] = [] 98 if strict_module_lockfile: 99 strict_lockfile.append('--lockfile_mode=error') 100 101 remote_cache: list[str] = [] 102 if use_remote_cache and ctx.luci: 103 remote_cache.append('--config=remote_cache') 104 if ctx.luci.is_ci: 105 # Only CI builders should attempt to write to the cache. Try 106 # builders will be denied permission if they do so. 107 remote_cache.append('--remote_upload_local_results=true') 108 109 symlink_prefix: list[str] = [] 110 if cmd != 'query': 111 # bazel query doesn't support the --symlink_prefix flag. 112 symlink_prefix.append(f'--symlink_prefix={ctx.output_dir / "bazel-"}') 113 114 ctx.output_dir.mkdir(exist_ok=True, parents=True) 115 try: 116 with contextlib.ExitStack() as stack: 117 if not stdout: 118 stdout = stack.enter_context( 119 (ctx.output_dir / f'bazel.{cmd}.stdout').open('w') 120 ) 121 122 with (ctx.output_dir / 'bazel.output.base').open('w') as outs, ( 123 ctx.output_dir / 'bazel.output.base.err' 124 ).open('w') as errs: 125 call( 126 BAZEL_EXECUTABLE, 127 'info', 128 'output_base', 129 tee=outs, 130 stderr=errs, 131 ) 132 133 call( 134 BAZEL_EXECUTABLE, 135 cmd, 136 *symlink_prefix, 137 *num_jobs, 138 *keep_going, 139 *strict_lockfile, 140 *remote_cache, 141 *args, 142 cwd=ctx.root, 143 tee=stdout, 144 call_annotation={'build_system': 'bazel'}, 145 **kwargs, 146 ) 147 148 except PresubmitFailure as exc: 149 if stdout: 150 failure = bazel_parser.parse_bazel_stdout(Path(stdout.name)) 151 if failure: 152 with ctx.failure_summary_log.open('w') as outs: 153 outs.write(failure) 154 155 raise exc 156 157 158def _gn_value(value) -> str: 159 if isinstance(value, bool): 160 return str(value).lower() 161 162 if ( 163 isinstance(value, str) 164 and '"' not in value 165 and not value.startswith("{") 166 and not value.startswith("[") 167 ): 168 return f'"{value}"' 169 170 if isinstance(value, (list, tuple)): 171 return f'[{", ".join(_gn_value(a) for a in value)}]' 172 173 # Fall-back case handles integers as well as strings that already 174 # contain double quotation marks, or look like scopes or lists. 175 return str(value) 176 177 178def gn_args_list(**kwargs) -> list[str]: 179 """Return a list of formatted strings to use as gn args. 180 181 Currently supports bool, int, and str values. In the case of str values, 182 quotation marks will be added automatically, unless the string already 183 contains one or more double quotation marks, or starts with a { or [ 184 character, in which case it will be passed through as-is. 185 """ 186 transformed_args = [] 187 for arg, val in kwargs.items(): 188 transformed_args.append(f'{arg}={_gn_value(val)}') 189 190 # Use ccache if available for faster repeat presubmit runs. 191 if which('ccache') and 'pw_command_launcher' not in kwargs: 192 transformed_args.append('pw_command_launcher="ccache"') 193 194 return transformed_args 195 196 197def gn_args(**kwargs) -> str: 198 """Builds a string to use for the --args argument to gn gen. 199 200 Currently supports bool, int, and str values. In the case of str values, 201 quotation marks will be added automatically, unless the string already 202 contains one or more double quotation marks, or starts with a { or [ 203 character, in which case it will be passed through as-is. 204 """ 205 return '--args=' + ' '.join(gn_args_list(**kwargs)) 206 207 208def write_gn_args_file(destination_file: Path, **kwargs) -> str: 209 """Write gn args to a file. 210 211 Currently supports bool, int, and str values. In the case of str values, 212 quotation marks will be added automatically, unless the string already 213 contains one or more double quotation marks, or starts with a { or [ 214 character, in which case it will be passed through as-is. 215 216 Returns: 217 The contents of the written file. 218 """ 219 contents = '\n'.join(gn_args_list(**kwargs)) 220 # Add a trailing linebreak 221 contents += '\n' 222 destination_file.parent.mkdir(exist_ok=True, parents=True) 223 224 if ( 225 destination_file.is_file() 226 and destination_file.read_text(encoding='utf-8') == contents 227 ): 228 # File is identical, don't re-write. 229 return contents 230 231 destination_file.write_text(contents, encoding='utf-8') 232 return contents 233 234 235def gn_gen( 236 ctx: PresubmitContext, 237 *args: str, 238 gn_check: bool = True, # pylint: disable=redefined-outer-name 239 gn_fail_on_unused: bool = True, 240 export_compile_commands: bool | str = True, 241 preserve_args_gn: bool = False, 242 **gn_arguments, 243) -> None: 244 """Runs gn gen in the specified directory with optional GN args. 245 246 Runs with --check=system if gn_check=True. Note that this does not cover 247 generated files. Run gn_check() after building to check generated files. 248 """ 249 all_gn_args = {'pw_build_COLORIZE_OUTPUT': pw_cli.color.is_enabled()} 250 all_gn_args.update(gn_arguments) 251 all_gn_args.update(ctx.override_gn_args) 252 _LOG.debug('%r', all_gn_args) 253 args_option = gn_args(**all_gn_args) 254 255 if not ctx.dry_run and not preserve_args_gn: 256 # Delete args.gn to ensure this is a clean build. 257 args_gn = ctx.output_dir / 'args.gn' 258 if args_gn.is_file(): 259 args_gn.unlink() 260 261 export_commands_arg = '' 262 if export_compile_commands: 263 export_commands_arg = '--export-compile-commands' 264 if isinstance(export_compile_commands, str): 265 export_commands_arg += f'={export_compile_commands}' 266 267 call( 268 'gn', 269 '--color' if pw_cli.color.is_enabled() else '--nocolor', 270 'gen', 271 ctx.output_dir, 272 *(['--check=system'] if gn_check else []), 273 *(['--fail-on-unused-args'] if gn_fail_on_unused else []), 274 *([export_commands_arg] if export_commands_arg else []), 275 *args, 276 *([args_option] if all_gn_args else []), 277 cwd=ctx.root, 278 call_annotation={ 279 'gn_gen_args': all_gn_args, 280 'gn_gen_args_option': args_option, 281 }, 282 ) 283 284 285def gn_check(ctx: PresubmitContext) -> PresubmitResult: 286 """Runs gn check, including on generated and system files.""" 287 call( 288 'gn', 289 'check', 290 ctx.output_dir, 291 '--check-generated', 292 '--check-system', 293 cwd=ctx.root, 294 ) 295 return PresubmitResult.PASS 296 297 298def ninja( 299 ctx: PresubmitContext, 300 *args, 301 save_compdb: bool = True, 302 save_graph: bool = True, 303 **kwargs, 304) -> None: 305 """Runs ninja in the specified directory.""" 306 307 num_jobs: list[str] = [] 308 if ctx.num_jobs is not None: 309 num_jobs.extend(('-j', str(ctx.num_jobs))) 310 311 keep_going: list[str] = [] 312 if ctx.continue_after_build_error: 313 keep_going.extend(('-k', '0')) 314 315 if save_compdb: 316 proc = log_run( 317 ['ninja', '-C', ctx.output_dir, '-t', 'compdb', *args], 318 capture_output=True, 319 **kwargs, 320 ) 321 if not ctx.dry_run: 322 (ctx.output_dir / 'ninja.compdb').write_bytes(proc.stdout) 323 324 if save_graph: 325 proc = log_run( 326 ['ninja', '-C', ctx.output_dir, '-t', 'graph', *args], 327 capture_output=True, 328 **kwargs, 329 ) 330 if not ctx.dry_run: 331 (ctx.output_dir / 'ninja.graph').write_bytes(proc.stdout) 332 333 ninja_stdout = ctx.output_dir / 'ninja.stdout' 334 ctx.output_dir.mkdir(exist_ok=True, parents=True) 335 try: 336 with ninja_stdout.open('w') as outs: 337 if sys.platform == 'win32': 338 # Windows doesn't support pw-wrap-ninja. 339 ninja_command = ['ninja'] 340 else: 341 ninja_command = ['pw-wrap-ninja', '--log-actions'] 342 343 call( 344 *ninja_command, 345 '-C', 346 ctx.output_dir, 347 *num_jobs, 348 *keep_going, 349 *args, 350 tee=outs, 351 propagate_sigterm=True, 352 call_annotation={'build_system': 'ninja'}, 353 **kwargs, 354 ) 355 356 except PresubmitFailure as exc: 357 failure = ninja_parser.parse_ninja_stdout(ninja_stdout) 358 if failure: 359 with ctx.failure_summary_log.open('w') as outs: 360 outs.write(failure) 361 362 raise exc 363 364 365def get_gn_args(directory: Path) -> list[dict[str, dict[str, str]]]: 366 """Dumps GN variables to JSON.""" 367 proc = log_run( 368 ['gn', 'args', directory, '--list', '--json'], stdout=subprocess.PIPE 369 ) 370 return json.loads(proc.stdout) 371 372 373def cmake( 374 ctx: PresubmitContext, 375 *args: str, 376 env: Mapping['str', 'str'] | None = None, 377) -> None: 378 """Runs CMake for Ninja on the given source and output directories.""" 379 call( 380 'cmake', 381 '-B', 382 ctx.output_dir, 383 '-S', 384 ctx.root, 385 '-G', 386 'Ninja', 387 *args, 388 env=env, 389 ) 390 391 392def env_with_clang_vars() -> Mapping[str, str]: 393 """Returns the environment variables with CC, CXX, etc. set for clang.""" 394 env = os.environ.copy() 395 env['CC'] = env['LD'] = env['AS'] = 'clang' 396 env['CXX'] = 'clang++' 397 return env 398 399 400def _get_paths_from_command(source_dir: Path, *args, **kwargs) -> Set[Path]: 401 """Runs a command and reads Bazel or GN //-style paths from it.""" 402 process = log_run(args, capture_output=True, cwd=source_dir, **kwargs) 403 404 if process.returncode: 405 _LOG.error( 406 'Build invocation failed with return code %d!', process.returncode 407 ) 408 _LOG.error( 409 '[COMMAND] %s\n%s\n%s', 410 *format_command(args, kwargs), 411 process.stderr.decode(), 412 ) 413 raise PresubmitFailure 414 415 files = set() 416 417 for line in process.stdout.splitlines(): 418 path = line.strip().lstrip(b'/').replace(b':', b'/').decode() 419 path = source_dir.joinpath(path) 420 if path.is_file(): 421 files.add(path) 422 423 return files 424 425 426# Finds string literals with '.' in them. 427_MAYBE_A_PATH = re.compile( 428 r'"' # Starting double quote. 429 # Start capture group 1 - the whole filename: 430 # File basename, a single period, file extension. 431 r'([^\n" ]+\.[^\n" ]+)' 432 # Non-capturing group 2 (optional). 433 r'(?: > [^\n"]+)?' # pw_zip style string "input_file.txt > output_file.txt" 434 r'"' # Ending double quote. 435) 436 437 438def _search_files_for_paths(build_files: Iterable[Path]) -> Iterable[Path]: 439 for build_file in build_files: 440 directory = build_file.parent 441 442 for string in _MAYBE_A_PATH.finditer(build_file.read_text()): 443 path = directory / string.group(1) 444 if path.is_file(): 445 yield path 446 447 448def _read_compile_commands(compile_commands: Path) -> dict: 449 with compile_commands.open('rb') as fd: 450 return json.load(fd) 451 452 453def compiled_files(compile_commands: Path) -> Iterable[Path]: 454 for command in _read_compile_commands(compile_commands): 455 file = Path(command['file']) 456 if file.is_absolute(): 457 yield file 458 else: 459 yield file.joinpath(command['directory']).resolve() 460 461 462def check_compile_commands_for_files( 463 compile_commands: Path | Iterable[Path], 464 files: Iterable[Path], 465 extensions: Collection[str] = format_code.CPP_SOURCE_EXTS, 466) -> list[Path]: 467 """Checks for paths in one or more compile_commands.json files. 468 469 Only checks C and C++ source files by default. 470 """ 471 if isinstance(compile_commands, Path): 472 compile_commands = [compile_commands] 473 474 compiled = frozenset( 475 itertools.chain.from_iterable( 476 compiled_files(cmds) for cmds in compile_commands 477 ) 478 ) 479 return [f for f in files if f not in compiled and f.suffix in extensions] 480 481 482def check_bazel_build_for_files( 483 bazel_extensions_to_check: Container[str], 484 files: Iterable[Path], 485 bazel_dirs: Iterable[Path] = (), 486) -> list[Path]: 487 """Checks that source files are in the Bazel builds. 488 489 Args: 490 bazel_extensions_to_check: which file suffixes to look for in Bazel 491 files: the files that should be checked 492 bazel_dirs: directories in which to run bazel query 493 494 Returns: 495 a list of missing files; will be empty if there were no missing files 496 """ 497 498 # Collect all paths in the Bazel builds. 499 bazel_builds: Set[Path] = set() 500 for directory in bazel_dirs: 501 bazel_builds.update( 502 _get_paths_from_command( 503 directory, 504 BAZEL_EXECUTABLE, 505 'query', 506 'kind("source file", //...:*)', 507 ) 508 ) 509 510 missing: list[Path] = [] 511 512 if bazel_dirs: 513 for path in (p for p in files if p.suffix in bazel_extensions_to_check): 514 if path not in bazel_builds: 515 # TODO: b/234883555 - Replace this workaround for fuzzers. 516 if 'fuzz' not in str(path): 517 missing.append(path) 518 519 if missing: 520 _LOG.warning( 521 '%s missing from the Bazel build:\n%s', 522 plural(missing, 'file', are=True), 523 '\n'.join(str(x) for x in missing), 524 ) 525 526 return missing 527 528 529def check_gn_build_for_files( 530 gn_extensions_to_check: Container[str], 531 files: Iterable[Path], 532 gn_dirs: Iterable[tuple[Path, Path]] = (), 533 gn_build_files: Iterable[Path] = (), 534) -> list[Path]: 535 """Checks that source files are in the GN build. 536 537 Args: 538 gn_extensions_to_check: which file suffixes to look for in GN 539 files: the files that should be checked 540 gn_dirs: (source_dir, output_dir) tuples with which to run gn desc 541 gn_build_files: paths to BUILD.gn files to directly search for paths 542 543 Returns: 544 a list of missing files; will be empty if there were no missing files 545 """ 546 547 # Collect all paths in GN builds. 548 gn_builds: Set[Path] = set() 549 550 for source_dir, output_dir in gn_dirs: 551 gn_builds.update( 552 _get_paths_from_command(source_dir, 'gn', 'desc', output_dir, '*') 553 ) 554 555 gn_builds.update(_search_files_for_paths(gn_build_files)) 556 557 missing: list[Path] = [] 558 559 if gn_dirs or gn_build_files: 560 for path in (p for p in files if p.suffix in gn_extensions_to_check): 561 if path not in gn_builds: 562 missing.append(path) 563 564 if missing: 565 _LOG.warning( 566 '%s missing from the GN build:\n%s', 567 plural(missing, 'file', are=True), 568 '\n'.join(str(x) for x in missing), 569 ) 570 571 return missing 572 573 574def check_soong_build_for_files( 575 soong_extensions_to_check: Container[str], 576 files: Iterable[Path], 577 soong_build_files: Iterable[Path] = (), 578) -> list[Path]: 579 """Checks that source files are in the Soong build. 580 581 Args: 582 bp_extensions_to_check: which file suffixes to look for in Soong files 583 files: the files that should be checked 584 bp_build_files: paths to Android.bp files to directly search for paths 585 586 Returns: 587 a list of missing files; will be empty if there were no missing files 588 """ 589 590 # Collect all paths in Soong builds. 591 soong_builds = set(_search_files_for_paths(soong_build_files)) 592 593 missing: list[Path] = [] 594 595 if soong_build_files: 596 for path in (p for p in files if p.suffix in soong_extensions_to_check): 597 if path not in soong_builds: 598 missing.append(path) 599 600 if missing: 601 _LOG.warning( 602 '%s missing from the Soong build:\n%s', 603 plural(missing, 'file', are=True), 604 '\n'.join(str(x) for x in missing), 605 ) 606 607 return missing 608 609 610def check_builds_for_files( 611 bazel_extensions_to_check: Container[str], 612 gn_extensions_to_check: Container[str], 613 files: Iterable[Path], 614 bazel_dirs: Iterable[Path] = (), 615 gn_dirs: Iterable[tuple[Path, Path]] = (), 616 gn_build_files: Iterable[Path] = (), 617) -> dict[str, list[Path]]: 618 """Checks that source files are in the GN and Bazel builds. 619 620 Args: 621 bazel_extensions_to_check: which file suffixes to look for in Bazel 622 gn_extensions_to_check: which file suffixes to look for in GN 623 files: the files that should be checked 624 bazel_dirs: directories in which to run bazel query 625 gn_dirs: (source_dir, output_dir) tuples with which to run gn desc 626 gn_build_files: paths to BUILD.gn files to directly search for paths 627 628 Returns: 629 a dictionary mapping build system ('Bazel' or 'GN' to a list of missing 630 files; will be empty if there were no missing files 631 """ 632 633 bazel_missing = check_bazel_build_for_files( 634 bazel_extensions_to_check=bazel_extensions_to_check, 635 files=files, 636 bazel_dirs=bazel_dirs, 637 ) 638 gn_missing = check_gn_build_for_files( 639 gn_extensions_to_check=gn_extensions_to_check, 640 files=files, 641 gn_dirs=gn_dirs, 642 gn_build_files=gn_build_files, 643 ) 644 645 result = {} 646 if bazel_missing: 647 result['Bazel'] = bazel_missing 648 if gn_missing: 649 result['GN'] = gn_missing 650 return result 651 652 653@contextlib.contextmanager 654def test_server(executable: str, output_dir: Path): 655 """Context manager that runs a test server executable. 656 657 Args: 658 executable: name of the test server executable 659 output_dir: path to the output directory (for logs) 660 """ 661 662 with open(output_dir / 'test_server.log', 'w') as outs: 663 try: 664 proc = subprocess.Popen( 665 [executable, '--verbose'], 666 stdout=outs, 667 stderr=subprocess.STDOUT, 668 ) 669 670 yield 671 672 finally: 673 proc.terminate() # pylint: disable=used-before-assignment 674 675 676@contextlib.contextmanager 677def modified_env(**envvars): 678 """Context manager that sets environment variables. 679 680 Use by assigning values to variable names in the argument list, e.g.: 681 `modified_env(MY_FLAG="some value")` 682 683 Args: 684 envvars: Keyword arguments 685 """ 686 saved_env = os.environ.copy() 687 os.environ.update(envvars) 688 try: 689 yield 690 finally: 691 os.environ = saved_env 692 693 694def fuzztest_prng_seed(ctx: PresubmitContext) -> str: 695 """Convert the RNG seed to the format expected by FuzzTest. 696 697 FuzzTest can be configured to use the seed by setting the 698 `FUZZTEST_PRNG_SEED` environment variable to this value. 699 700 Args: 701 ctx: The context that includes a pseudorandom number generator seed. 702 """ 703 rng_bytes = ctx.rng_seed.to_bytes(32, sys.byteorder) 704 return base64.urlsafe_b64encode(rng_bytes).decode('ascii').rstrip('=') 705 706 707@filter_paths( 708 file_filter=FileFilter( 709 endswith=('.bzl', '.bazel'), 710 name=('WORKSPACE',), 711 exclude=(r'pw_presubmit/py/pw_presubmit/format/test_data',), 712 ) 713) 714def bazel_lint(ctx: PresubmitContext): 715 """Runs buildifier with lint on Bazel files. 716 717 Should be run after bazel_format since that will give more useful output 718 for formatting-only issues. 719 """ 720 721 failure = False 722 for path in ctx.paths: 723 try: 724 call('buildifier', '--lint=warn', '--mode=check', path) 725 except PresubmitFailure: 726 failure = True 727 728 if failure: 729 raise PresubmitFailure 730 731 732@Check 733def gn_gen_check(ctx: PresubmitContext): 734 """Runs gn gen --check to enforce correct header dependencies.""" 735 gn_gen(ctx, gn_check=True) 736 737 738Item = int | str 739Value = Item | Sequence[Item] 740ValueCallable = Callable[[PresubmitContext], Value] 741InputItem = Item | ValueCallable 742InputValue = InputItem | Sequence[InputItem] 743 744 745def _value(ctx: PresubmitContext, val: InputValue) -> Value: 746 """Process any lambdas inside val 747 748 val is a single value or a list of values, any of which might be a lambda 749 that needs to be resolved. Call each of these lambdas with ctx and replace 750 the lambda with the result. Return the updated top-level structure. 751 """ 752 753 if isinstance(val, (str, int)): 754 return val 755 if callable(val): 756 return val(ctx) 757 758 result: list[Item] = [] 759 for item in val: 760 if callable(item): 761 call_result = item(ctx) 762 if isinstance(call_result, (int, str)): 763 result.append(call_result) 764 else: # Sequence. 765 result.extend(call_result) 766 elif isinstance(item, (int, str)): 767 result.append(item) 768 else: # Sequence. 769 result.extend(item) 770 return result 771 772 773_CtxMgrLambda = Callable[[PresubmitContext], ContextManager] 774_CtxMgrOrLambda = ContextManager | _CtxMgrLambda 775 776 777@dataclass(frozen=True) 778class CommonCoverageOptions: 779 """Coverage options shared by both CodeSearch and Gerrit. 780 781 For Google use only. 782 """ 783 784 # The "root" of the Kalypsi GCS bucket path to which the coverage data 785 # should be uploaded. Typically gs://ng3-metrics/ng3-<teamname>-coverage. 786 target_bucket_root: str 787 788 # The project name in the Kalypsi GCS bucket path. 789 target_bucket_project: str 790 791 # See go/kalypsi-abs#trace-type-required. 792 trace_type: str 793 794 # go/kalypsi-abs#owner-required. 795 owner: str 796 797 # go/kalypsi-abs#bug-component-required. 798 bug_component: str 799 800 801@dataclass(frozen=True) 802class CodeSearchCoverageOptions: 803 """CodeSearch-specific coverage options. For Google use only.""" 804 805 # The name of the Gerrit host containing the CodeSearch repo. Just the name 806 # ("pigweed"), not the full URL ("pigweed.googlesource.com"). This may be 807 # different from the host from which the code was originally checked out. 808 host: str 809 810 # The name of the project, as expected by CodeSearch. Typically 811 # 'codesearch'. 812 project: str 813 814 # See go/kalypsi-abs#ref-required. 815 ref: str 816 817 # See go/kalypsi-abs#source-required. 818 source: str 819 820 # See go/kalypsi-abs#add-prefix-optional. 821 add_prefix: str = '' 822 823 824@dataclass(frozen=True) 825class GerritCoverageOptions: 826 """Gerrit-specific coverage options. For Google use only.""" 827 828 # The name of the project, as expected by Gerrit. This is typically the 829 # repository name, e.g. 'pigweed/pigweed' for upstream Pigweed. 830 # See go/kalypsi-inc#project-required. 831 project: str 832 833 834@dataclass(frozen=True) 835class CoverageOptions: 836 """Coverage collection configuration. For Google use only.""" 837 838 common: CommonCoverageOptions 839 codesearch: tuple[CodeSearchCoverageOptions, ...] 840 gerrit: GerritCoverageOptions 841 842 843class _NinjaBase(Check): 844 """Thin wrapper of Check for steps that call ninja.""" 845 846 def __init__( 847 self, 848 *args, 849 packages: Sequence[str] = (), 850 ninja_contexts: Sequence[_CtxMgrOrLambda] = (), 851 ninja_targets: str | Sequence[str] | Sequence[Sequence[str]] = (), 852 coverage_options: CoverageOptions | None = None, 853 **kwargs, 854 ): 855 """Initializes a _NinjaBase object. 856 857 Args: 858 *args: Passed on to superclass. 859 packages: List of 'pw package' packages to install. 860 ninja_contexts: List of context managers to apply around ninja 861 calls. 862 ninja_targets: Single ninja target, list of Ninja targets, or list 863 of list of ninja targets. If a list of a list, ninja will be 864 called multiple times with the same build directory. 865 coverage_options: Coverage collection options (or None, if not 866 collecting coverage data). 867 **kwargs: Passed on to superclass. 868 """ 869 super().__init__(*args, **kwargs) 870 self._packages: Sequence[str] = packages 871 self._ninja_contexts: tuple[_CtxMgrOrLambda, ...] = tuple( 872 ninja_contexts 873 ) 874 self._coverage_options = coverage_options 875 876 if isinstance(ninja_targets, str): 877 ninja_targets = (ninja_targets,) 878 ninja_targets = list(ninja_targets) 879 all_strings = all(isinstance(x, str) for x in ninja_targets) 880 any_strings = any(isinstance(x, str) for x in ninja_targets) 881 if ninja_targets and all_strings != any_strings: 882 raise ValueError(repr(ninja_targets)) 883 884 self._ninja_target_lists: tuple[tuple[str, ...], ...] 885 if all_strings: 886 targets: list[str] = [] 887 for target in ninja_targets: 888 targets.append(target) # type: ignore 889 self._ninja_target_lists = (tuple(targets),) 890 else: 891 self._ninja_target_lists = tuple(tuple(x) for x in ninja_targets) 892 893 @property 894 def ninja_targets(self) -> list[str]: 895 return list(itertools.chain(*self._ninja_target_lists)) 896 897 def _install_package( # pylint: disable=no-self-use 898 self, 899 ctx: PresubmitContext, 900 package: str, 901 ) -> PresubmitResult: 902 install_package(ctx, package) 903 return PresubmitResult.PASS 904 905 @contextlib.contextmanager 906 def _context(self, ctx: PresubmitContext): 907 """Apply any context managers necessary for building.""" 908 with contextlib.ExitStack() as stack: 909 for mgr in self._ninja_contexts: 910 if isinstance(mgr, contextlib.AbstractContextManager): 911 stack.enter_context(mgr) 912 else: 913 stack.enter_context(mgr(ctx)) # type: ignore 914 yield 915 916 def _ninja( 917 self, ctx: PresubmitContext, targets: Sequence[str] 918 ) -> PresubmitResult: 919 with self._context(ctx): 920 ninja(ctx, *targets) 921 return PresubmitResult.PASS 922 923 def _coverage( 924 self, ctx: PresubmitContext, options: CoverageOptions 925 ) -> PresubmitResult: 926 """Archive and (on LUCI) upload coverage reports.""" 927 reports = ctx.output_dir / 'coverage_reports' 928 os.makedirs(reports, exist_ok=True) 929 coverage_jsons: list[Path] = [] 930 for path in ctx.output_dir.rglob('coverage_report'): 931 _LOG.debug('exploring %s', path) 932 name = str(path.relative_to(ctx.output_dir)) 933 name = name.replace('_', '').replace('/', '_') 934 with tarfile.open(reports / f'{name}.tar.gz', 'w:gz') as tar: 935 tar.add(path, arcname=name, recursive=True) 936 json_path = path / 'json' / 'report.json' 937 if json_path.is_file(): 938 _LOG.debug('found json %s', json_path) 939 coverage_jsons.append(json_path) 940 941 if not coverage_jsons: 942 ctx.fail('No coverage json file found') 943 return PresubmitResult.FAIL 944 945 if len(coverage_jsons) > 1: 946 _LOG.warning( 947 'More than one coverage json file, selecting first: %r', 948 coverage_jsons, 949 ) 950 951 coverage_json = coverage_jsons[0] 952 953 if ctx.luci: 954 if not ctx.luci.is_prod: 955 _LOG.warning('Not uploading coverage since not running in prod') 956 return PresubmitResult.PASS 957 958 with self._context(ctx): 959 metadata_json_paths = _write_coverage_metadata(ctx, options) 960 for i, metadata_json in enumerate(metadata_json_paths): 961 # GCS bucket paths are POSIX-like. 962 coverage_gcs_path = posixpath.join( 963 options.common.target_bucket_root, 964 'incremental' if ctx.luci.is_try else 'absolute', 965 options.common.target_bucket_project, 966 f'{ctx.luci.buildbucket_id}-{i}', 967 ) 968 _copy_to_gcs( 969 ctx, 970 coverage_json, 971 posixpath.join(coverage_gcs_path, 'report.json'), 972 ) 973 _copy_to_gcs( 974 ctx, 975 metadata_json, 976 posixpath.join(coverage_gcs_path, 'metadata.json'), 977 ) 978 979 return PresubmitResult.PASS 980 981 _LOG.warning('Not uploading coverage since running locally') 982 return PresubmitResult.PASS 983 984 def _package_substeps(self) -> Iterator[SubStep]: 985 for package in self._packages: 986 yield SubStep( 987 f'install {package} package', 988 self._install_package, 989 (package,), 990 ) 991 992 def _ninja_substeps(self) -> Iterator[SubStep]: 993 targets_parts = set() 994 for targets in self._ninja_target_lists: 995 targets_part = " ".join(targets) 996 maxlen = 70 997 if len(targets_part) > maxlen: 998 targets_part = f'{targets_part[0:maxlen-3]}...' 999 assert targets_part not in targets_parts 1000 targets_parts.add(targets_part) 1001 yield SubStep(f'ninja {targets_part}', self._ninja, (targets,)) 1002 1003 def _coverage_substeps(self) -> Iterator[SubStep]: 1004 if self._coverage_options is not None: 1005 yield SubStep('coverage', self._coverage, (self._coverage_options,)) 1006 1007 1008def _copy_to_gcs(ctx: PresubmitContext, filepath: Path, gcs_dst: str): 1009 cmd = [ 1010 "gsutil", 1011 "cp", 1012 filepath, 1013 gcs_dst, 1014 ] 1015 1016 upload_stdout = ctx.output_dir / (filepath.name + '.stdout') 1017 with upload_stdout.open('w') as outs: 1018 call(*cmd, tee=outs) 1019 1020 1021def _write_coverage_metadata( 1022 ctx: PresubmitContext, options: CoverageOptions 1023) -> Sequence[Path]: 1024 """Write out Kalypsi coverage metadata file(s) and return their paths.""" 1025 assert ctx.luci is not None 1026 assert len(ctx.luci.triggers) == 1 1027 change = ctx.luci.triggers[0] 1028 1029 metadata = { 1030 'trace_type': options.common.trace_type, 1031 'trim_prefix': str(ctx.root), 1032 'patchset_num': change.patchset, 1033 'change_id': change.number, 1034 'owner': options.common.owner, 1035 'bug_component': options.common.bug_component, 1036 } 1037 1038 if ctx.luci.is_try: 1039 # Running in CQ: uploading incremental coverage 1040 metadata.update( 1041 { 1042 'change_id': change.number, 1043 'host': change.gerrit_name, 1044 'patchset_num': change.patchset, 1045 'project': options.gerrit.project, 1046 } 1047 ) 1048 1049 metadata_json = ctx.output_dir / "metadata.json" 1050 with metadata_json.open('w') as metadata_file: 1051 json.dump(metadata, metadata_file) 1052 return (metadata_json,) 1053 1054 # Running in CI: uploading absolute coverage, possibly to multiple locations 1055 # since a repo could be in codesearch in multiple places. 1056 metadata_jsons = [] 1057 for i, cs in enumerate(options.codesearch): 1058 metadata.update( 1059 { 1060 'add_prefix': cs.add_prefix, 1061 'commit_id': change.ref, 1062 'host': cs.host, 1063 'project': cs.project, 1064 'ref': cs.ref, 1065 'source': cs.source, 1066 } 1067 ) 1068 1069 metadata_json = ctx.output_dir / f'metadata-{i}.json' 1070 with metadata_json.open('w') as metadata_file: 1071 json.dump(metadata, metadata_file) 1072 metadata_jsons.append(metadata_json) 1073 1074 return tuple(metadata_jsons) 1075 1076 1077class GnGenNinja(_NinjaBase): 1078 """Thin wrapper of Check for steps that just call gn/ninja. 1079 1080 Runs gn gen, ninja, then gn check. 1081 """ 1082 1083 def __init__( 1084 self, 1085 *args, 1086 gn_args: ( # pylint: disable=redefined-outer-name 1087 dict[str, Any] | None 1088 ) = None, 1089 **kwargs, 1090 ): 1091 """Initializes a GnGenNinja object. 1092 1093 Args: 1094 *args: Passed on to superclass. 1095 gn_args: dict of GN args. 1096 **kwargs: Passed on to superclass. 1097 """ 1098 super().__init__(self._substeps(), *args, **kwargs) 1099 self._gn_args: dict[str, Any] = gn_args or {} 1100 1101 def add_default_gn_args(self, args): 1102 """Add any project-specific default GN args to 'args'.""" 1103 1104 @property 1105 def gn_args(self) -> dict[str, Any]: 1106 return self._gn_args 1107 1108 def _gn_gen(self, ctx: PresubmitContext) -> PresubmitResult: 1109 args: dict[str, Any] = {} 1110 if self._coverage_options is not None: 1111 args['pw_toolchain_COVERAGE_ENABLED'] = True 1112 args['pw_build_PYTHON_TEST_COVERAGE'] = True 1113 1114 if ctx.incremental: 1115 args['pw_toolchain_PROFILE_SOURCE_FILES'] = [ 1116 f'//{x.relative_to(ctx.root)}' for x in ctx.paths 1117 ] 1118 1119 self.add_default_gn_args(args) 1120 1121 args.update({k: _value(ctx, v) for k, v in self._gn_args.items()}) 1122 gn_gen(ctx, gn_check=False, **args) # type: ignore 1123 return PresubmitResult.PASS 1124 1125 def _substeps(self) -> Iterator[SubStep]: 1126 yield from self._package_substeps() 1127 1128 yield SubStep('gn gen', self._gn_gen) 1129 1130 yield from self._ninja_substeps() 1131 1132 # Run gn check after building so it can check generated files. 1133 yield SubStep('gn check', gn_check) 1134 1135 yield from self._coverage_substeps() 1136