1# Copyright 2023 The Pigweed Authors 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); you may not 4# use this file except in compliance with the License. You may obtain a copy of 5# the License at 6# 7# https://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 12# License for the specific language governing permissions and limitations under 13# the License. 14"""pw_presubmit ContextVar.""" 15 16from __future__ import annotations 17 18from contextvars import ContextVar 19import dataclasses 20import enum 21import inspect 22import logging 23import json 24import os 25from pathlib import Path 26import re 27import shlex 28import shutil 29import subprocess 30import tempfile 31from typing import ( 32 Any, 33 Iterable, 34 NamedTuple, 35 Sequence, 36 TYPE_CHECKING, 37) 38import urllib 39 40import pw_cli.color 41import pw_cli.env 42import pw_env_setup.config_file 43 44if TYPE_CHECKING: 45 from pw_presubmit.presubmit import Check 46 47_COLOR = pw_cli.color.colors() 48_LOG: logging.Logger = logging.getLogger(__name__) 49 50PRESUBMIT_CHECK_TRACE: ContextVar[ 51 dict[str, list[PresubmitCheckTrace]] 52] = ContextVar('pw_presubmit_check_trace', default={}) 53 54 55@dataclasses.dataclass(frozen=True) 56class FormatOptions: 57 python_formatter: str | None = 'black' 58 black_path: str | None = 'black' 59 exclude: Sequence[re.Pattern] = dataclasses.field(default_factory=list) 60 61 @staticmethod 62 def load(env: dict[str, str] | None = None) -> FormatOptions: 63 config = pw_env_setup.config_file.load(env=env) 64 fmt = config.get('pw', {}).get('pw_presubmit', {}).get('format', {}) 65 return FormatOptions( 66 python_formatter=fmt.get('python_formatter', 'black'), 67 black_path=fmt.get('black_path', 'black'), 68 exclude=tuple(re.compile(x) for x in fmt.get('exclude', ())), 69 ) 70 71 def filter_paths(self, paths: Iterable[Path]) -> tuple[Path, ...]: 72 root = Path(pw_cli.env.pigweed_environment().PW_PROJECT_ROOT) 73 relpaths = [x.relative_to(root) for x in paths] 74 75 for filt in self.exclude: 76 relpaths = [x for x in relpaths if not filt.search(str(x))] 77 return tuple(root / x for x in relpaths) 78 79 80def get_buildbucket_info(bbid) -> dict[str, Any]: 81 if not bbid or not shutil.which('bb'): 82 return {} 83 84 output = subprocess.check_output( 85 ['bb', 'get', '-json', '-p', f'{bbid}'], text=True 86 ) 87 return json.loads(output) 88 89 90@dataclasses.dataclass 91class LuciPipeline: 92 """Details of previous builds in this pipeline, if applicable. 93 94 Attributes: 95 round: The zero-indexed round number. 96 builds_from_previous_iteration: A list of the buildbucket ids from the 97 previous round, if any. 98 """ 99 100 round: int 101 builds_from_previous_iteration: Sequence[int] 102 103 @staticmethod 104 def create( 105 bbid: int, 106 fake_pipeline_props: dict[str, Any] | None = None, 107 ) -> LuciPipeline | None: 108 pipeline_props: dict[str, Any] 109 if fake_pipeline_props is not None: 110 pipeline_props = fake_pipeline_props 111 else: 112 pipeline_props = ( 113 get_buildbucket_info(bbid) 114 .get('input', {}) 115 .get('properties', {}) 116 .get('$pigweed/pipeline', {}) 117 ) 118 if not pipeline_props.get('inside_a_pipeline', False): 119 return None 120 121 return LuciPipeline( 122 round=int(pipeline_props['round']), 123 builds_from_previous_iteration=list( 124 int(x) for x in pipeline_props['builds_from_previous_iteration'] 125 ), 126 ) 127 128 129@dataclasses.dataclass 130class LuciTrigger: 131 """Details the pending change or submitted commit triggering the build. 132 133 Attributes: 134 number: The number of the change in Gerrit. 135 patchset: The number of the patchset of the change. 136 remote: The full URL of the remote. 137 project: The name of the project in Gerrit. 138 branch: The name of the branch on which this change is being/was 139 submitted. 140 ref: The "refs/changes/.." path that can be used to reference the 141 patch for unsubmitted changes and the hash for submitted changes. 142 gerrit_name: The name of the googlesource.com Gerrit host. 143 submitted: Whether the change has been submitted or is still pending. 144 gerrit_host: The scheme and hostname of the googlesource.com Gerrit 145 host. 146 gerrit_url: The full URL to this change on the Gerrit host. 147 gitiles_url: The full URL to this commit in Gitiles. 148 """ 149 150 number: int 151 patchset: int 152 remote: str 153 project: str 154 branch: str 155 ref: str 156 gerrit_name: str 157 submitted: bool 158 159 @property 160 def gerrit_host(self): 161 return f'https://{self.gerrit_name}-review.googlesource.com' 162 163 @property 164 def gerrit_url(self): 165 if not self.number: 166 return self.gitiles_url 167 return f'{self.gerrit_host}/c/{self.number}' 168 169 @property 170 def gitiles_url(self): 171 return f'{self.remote}/+/{self.ref}' 172 173 @staticmethod 174 def create_from_environment( 175 env: dict[str, str] | None = None, 176 ) -> Sequence['LuciTrigger']: 177 if not env: 178 env = os.environ.copy() 179 raw_path = env.get('TRIGGERING_CHANGES_JSON') 180 if not raw_path: 181 return () 182 path = Path(raw_path) 183 if not path.is_file(): 184 return () 185 186 result = [] 187 with open(path, 'r') as ins: 188 for trigger in json.load(ins): 189 keys = { 190 'number', 191 'patchset', 192 'remote', 193 'project', 194 'branch', 195 'ref', 196 'gerrit_name', 197 'submitted', 198 } 199 if keys <= trigger.keys(): 200 result.append(LuciTrigger(**{x: trigger[x] for x in keys})) 201 202 return tuple(result) 203 204 @staticmethod 205 def create_for_testing(**kwargs): 206 change = { 207 'number': 123456, 208 'patchset': 1, 209 'remote': 'https://pigweed.googlesource.com/pigweed/pigweed', 210 'project': 'pigweed/pigweed', 211 'branch': 'main', 212 'ref': 'refs/changes/56/123456/1', 213 'gerrit_name': 'pigweed', 214 'submitted': True, 215 } 216 change.update(kwargs) 217 218 with tempfile.TemporaryDirectory() as tempdir: 219 changes_json = Path(tempdir) / 'changes.json' 220 with changes_json.open('w') as outs: 221 json.dump([change], outs) 222 env = {'TRIGGERING_CHANGES_JSON': changes_json} 223 return LuciTrigger.create_from_environment(env) 224 225 226@dataclasses.dataclass 227class LuciContext: 228 """LUCI-specific information about the environment. 229 230 Attributes: 231 buildbucket_id: The globally-unique buildbucket id of the build. 232 build_number: The builder-specific incrementing build number, if 233 configured for this builder. 234 project: The LUCI project under which this build is running (often 235 "pigweed" or "pigweed-internal"). 236 bucket: The LUCI bucket under which this build is running (often ends 237 with "ci" or "try"). 238 builder: The builder being run. 239 swarming_server: The swarming server on which this build is running. 240 swarming_task_id: The swarming task id of this build. 241 cas_instance: The CAS instance accessible from this build. 242 context_file: The path to the LUCI_CONTEXT file. 243 pipeline: Information about the build pipeline, if applicable. 244 triggers: Information about triggering commits, if applicable. 245 is_try: True if the bucket is a try bucket. 246 is_ci: True if the bucket is a ci bucket. 247 is_dev: True if the bucket is a dev bucket. 248 is_shadow: True if the bucket is a shadow bucket. 249 is_prod: True if both is_dev and is_shadow are False. 250 """ 251 252 buildbucket_id: int 253 build_number: int 254 project: str 255 bucket: str 256 builder: str 257 swarming_server: str 258 swarming_task_id: str 259 cas_instance: str 260 context_file: Path 261 pipeline: LuciPipeline | None 262 triggers: Sequence[LuciTrigger] = dataclasses.field(default_factory=tuple) 263 264 @property 265 def is_try(self): 266 return re.search(r'\btry$', self.bucket) 267 268 @property 269 def is_ci(self): 270 return re.search(r'\bci$', self.bucket) 271 272 @property 273 def is_dev(self): 274 return re.search(r'\bdev\b', self.bucket) 275 276 @property 277 def is_shadow(self): 278 return re.search(r'\bshadow\b', self.bucket) 279 280 @property 281 def is_prod(self): 282 return not self.is_dev and not self.is_shadow 283 284 @staticmethod 285 def create_from_environment( 286 env: dict[str, str] | None = None, 287 fake_pipeline_props: dict[str, Any] | None = None, 288 ) -> LuciContext | None: 289 """Create a LuciContext from the environment.""" 290 291 if not env: 292 env = os.environ.copy() 293 294 luci_vars = [ 295 'BUILDBUCKET_ID', 296 'BUILDBUCKET_NAME', 297 'BUILD_NUMBER', 298 'LUCI_CONTEXT', 299 'SWARMING_TASK_ID', 300 'SWARMING_SERVER', 301 ] 302 if any(x for x in luci_vars if x not in env): 303 return None 304 305 project, bucket, builder = env['BUILDBUCKET_NAME'].split(':') 306 307 bbid: int = 0 308 pipeline: LuciPipeline | None = None 309 try: 310 bbid = int(env['BUILDBUCKET_ID']) 311 pipeline = LuciPipeline.create(bbid, fake_pipeline_props) 312 313 except ValueError: 314 pass 315 316 # Logic to identify cas instance from swarming server is derived from 317 # https://chromium.googlesource.com/infra/luci/recipes-py/+/main/recipe_modules/cas/api.py 318 swarm_server = env['SWARMING_SERVER'] 319 cas_project = urllib.parse.urlparse(swarm_server).netloc.split('.')[0] 320 cas_instance = f'projects/{cas_project}/instances/default_instance' 321 322 result = LuciContext( 323 buildbucket_id=bbid, 324 build_number=int(env['BUILD_NUMBER']), 325 project=project, 326 bucket=bucket, 327 builder=builder, 328 swarming_server=env['SWARMING_SERVER'], 329 swarming_task_id=env['SWARMING_TASK_ID'], 330 cas_instance=cas_instance, 331 pipeline=pipeline, 332 triggers=LuciTrigger.create_from_environment(env), 333 context_file=Path(env['LUCI_CONTEXT']), 334 ) 335 _LOG.debug('%r', result) 336 return result 337 338 @staticmethod 339 def create_for_testing(**kwargs): 340 env = { 341 'BUILDBUCKET_ID': '881234567890', 342 'BUILDBUCKET_NAME': 'pigweed:bucket.try:builder-name', 343 'BUILD_NUMBER': '123', 344 'LUCI_CONTEXT': '/path/to/context/file.json', 345 'SWARMING_SERVER': 'https://chromium-swarm.appspot.com', 346 'SWARMING_TASK_ID': 'cd2dac62d2', 347 } 348 env.update(kwargs) 349 350 return LuciContext.create_from_environment(env, {}) 351 352 353@dataclasses.dataclass 354class FormatContext: 355 """Context passed into formatting helpers. 356 357 This class is a subset of PresubmitContext containing only what's needed by 358 formatters. 359 360 For full documentation on the members see the PresubmitContext section of 361 pw_presubmit/docs.rst. 362 363 Attributes: 364 root: Source checkout root directory 365 output_dir: Output directory for this specific language. 366 paths: Modified files for the presubmit step to check (often used in 367 formatting steps but ignored in compile steps). 368 package_root: Root directory for pw package installations. 369 format_options: Formatting options, derived from pigweed.json. 370 dry_run: Whether to just report issues or also fix them. 371 """ 372 373 root: Path | None 374 output_dir: Path 375 paths: tuple[Path, ...] 376 package_root: Path 377 format_options: FormatOptions 378 dry_run: bool = False 379 380 def append_check_command(self, *command_args, **command_kwargs) -> None: 381 """Empty append_check_command.""" 382 383 384class PresubmitFailure(Exception): 385 """Optional exception to use for presubmit failures.""" 386 387 def __init__( 388 self, 389 description: str = '', 390 path: Path | None = None, 391 line: int | None = None, 392 ): 393 line_part: str = '' 394 if line is not None: 395 line_part = f'{line}:' 396 super().__init__( 397 f'{path}:{line_part} {description}' if path else description 398 ) 399 400 401@dataclasses.dataclass 402class PresubmitContext: # pylint: disable=too-many-instance-attributes 403 """Context passed into presubmit checks. 404 405 For full documentation on the members see pw_presubmit/docs.rst. 406 407 Attributes: 408 root: Source checkout root directory. 409 repos: Repositories (top-level and submodules) processed by 410 `pw presubmit`. 411 output_dir: Output directory for this specific presubmit step. 412 failure_summary_log: Path where steps should write a brief summary of 413 any failures encountered for use by other tooling. 414 paths: Modified files for the presubmit step to check (often used in 415 formatting steps but ignored in compile steps). 416 all_paths: All files in the tree. 417 package_root: Root directory for pw package installations. 418 override_gn_args: Additional GN args processed by `build.gn_gen()`. 419 luci: Information about the LUCI build or None if not running in LUCI. 420 format_options: Formatting options, derived from pigweed.json. 421 num_jobs: Number of jobs to run in parallel. 422 continue_after_build_error: For steps that compile, don't exit on the 423 first compilation error. 424 rng_seed: Seed for a random number generator, for the few steps that 425 need one. 426 full: Whether this is a full or incremental presubmit run. 427 _failed: Whether the presubmit step in question has failed. Set to True 428 by calling ctx.fail(). 429 dry_run: Whether to actually execute commands or just log them. 430 use_remote_cache: Whether to tell the build system to use RBE. 431 pw_root: The path to the Pigweed repository. 432 """ 433 434 root: Path 435 repos: tuple[Path, ...] 436 output_dir: Path 437 failure_summary_log: Path 438 paths: tuple[Path, ...] 439 all_paths: tuple[Path, ...] 440 package_root: Path 441 luci: LuciContext | None 442 override_gn_args: dict[str, str] 443 format_options: FormatOptions 444 num_jobs: int | None = None 445 continue_after_build_error: bool = False 446 rng_seed: int = 1 447 full: bool = False 448 _failed: bool = False 449 dry_run: bool = False 450 use_remote_cache: bool = False 451 pw_root: Path = pw_cli.env.pigweed_environment().PW_ROOT 452 453 @property 454 def failed(self) -> bool: 455 return self._failed 456 457 @property 458 def incremental(self) -> bool: 459 return not self.full 460 461 def fail( 462 self, 463 description: str, 464 path: Path | None = None, 465 line: int | None = None, 466 ): 467 """Add a failure to this presubmit step. 468 469 If this is called at least once the step fails, but not immediately—the 470 check is free to continue and possibly call this method again. 471 """ 472 _LOG.warning('%s', PresubmitFailure(description, path, line)) 473 self._failed = True 474 475 @staticmethod 476 def create_for_testing(**kwargs): 477 parsed_env = pw_cli.env.pigweed_environment() 478 root = parsed_env.PW_PROJECT_ROOT 479 presubmit_root = root / 'out' / 'presubmit' 480 presubmit_kwargs = { 481 'root': root, 482 'repos': (root,), 483 'output_dir': presubmit_root / 'test', 484 'failure_summary_log': presubmit_root / 'failure-summary.log', 485 'paths': (root / 'foo.cc', root / 'foo.py'), 486 'all_paths': (root / 'BUILD.gn', root / 'foo.cc', root / 'foo.py'), 487 'package_root': root / 'environment' / 'packages', 488 'luci': None, 489 'override_gn_args': {}, 490 'format_options': FormatOptions(), 491 } 492 presubmit_kwargs.update(kwargs) 493 return PresubmitContext(**presubmit_kwargs) 494 495 def append_check_command( 496 self, 497 *command_args, 498 call_annotation: dict[Any, Any] | None = None, 499 **command_kwargs, 500 ) -> None: 501 """Save a subprocess command annotation to this presubmit context. 502 503 This is used to capture commands that will be run for display in ``pw 504 presubmit --dry-run.`` 505 506 Args: 507 508 command_args: All args that would normally be passed to 509 subprocess.run 510 511 call_annotation: Optional key value pairs of data to save for this 512 command. Examples: 513 514 :: 515 516 call_annotation={'pw_package_install': 'teensy'} 517 call_annotation={'build_system': 'bazel'} 518 call_annotation={'build_system': 'ninja'} 519 520 command_kwargs: keyword args that would normally be passed to 521 subprocess.run. 522 """ 523 call_annotation = call_annotation if call_annotation else {} 524 calling_func: str | None = None 525 calling_check = None 526 527 # Loop through the current call stack looking for `self`, and stopping 528 # when self is a Check() instance and if the __call__ or _try_call 529 # functions are in the stack. 530 531 # This used to be an isinstance(obj, Check) call, but it was changed to 532 # this so Check wouldn't need to be imported here. Doing so would create 533 # a dependency loop. 534 def is_check_object(obj): 535 return getattr(obj, '_is_presubmit_check_object', False) 536 537 for frame_info in inspect.getouterframes(inspect.currentframe()): 538 self_obj = frame_info.frame.f_locals.get('self', None) 539 if ( 540 self_obj 541 and is_check_object(self_obj) 542 and frame_info.function in ['_try_call', '__call__'] 543 ): 544 calling_func = frame_info.function 545 calling_check = self_obj 546 547 save_check_trace( 548 self.output_dir, 549 PresubmitCheckTrace( 550 self, 551 calling_check, 552 calling_func, 553 command_args, 554 command_kwargs, 555 call_annotation, 556 ), 557 ) 558 559 def __post_init__(self) -> None: 560 PRESUBMIT_CONTEXT.set(self) 561 562 def __hash__(self): 563 return hash( 564 tuple( 565 tuple(attribute.items()) 566 if isinstance(attribute, dict) 567 else attribute 568 for attribute in dataclasses.astuple(self) 569 ) 570 ) 571 572 573PRESUBMIT_CONTEXT: ContextVar[PresubmitContext | None] = ContextVar( 574 'pw_presubmit_context', default=None 575) 576 577 578def get_presubmit_context(): 579 return PRESUBMIT_CONTEXT.get() 580 581 582class PresubmitCheckTraceType(enum.Enum): 583 BAZEL = 'BAZEL' 584 CMAKE = 'CMAKE' 585 GN_NINJA = 'GN_NINJA' 586 PW_PACKAGE = 'PW_PACKAGE' 587 588 589class PresubmitCheckTrace(NamedTuple): 590 ctx: PresubmitContext 591 check: Check | None 592 func: str | None 593 args: Iterable[Any] 594 kwargs: dict[Any, Any] 595 call_annotation: dict[Any, Any] 596 597 def __repr__(self) -> str: 598 return f'''CheckTrace( 599 ctx={self.ctx.output_dir} 600 id(ctx)={id(self.ctx)} 601 check={self.check} 602 args={self.args} 603 kwargs={self.kwargs.keys()} 604 call_annotation={self.call_annotation} 605)''' 606 607 608def save_check_trace(output_dir: Path, trace: PresubmitCheckTrace) -> None: 609 trace_key = str(output_dir.resolve()) 610 trace_list = PRESUBMIT_CHECK_TRACE.get().get(trace_key, []) 611 trace_list.append(trace) 612 PRESUBMIT_CHECK_TRACE.get()[trace_key] = trace_list 613 614 615def get_check_traces(ctx: PresubmitContext) -> list[PresubmitCheckTrace]: 616 trace_key = str(ctx.output_dir.resolve()) 617 return PRESUBMIT_CHECK_TRACE.get().get(trace_key, []) 618 619 620def log_check_traces(ctx: PresubmitContext) -> None: 621 traces = PRESUBMIT_CHECK_TRACE.get() 622 623 for _output_dir, check_traces in traces.items(): 624 for check_trace in check_traces: 625 if check_trace.ctx != ctx: 626 continue 627 628 quoted_command_args = ' '.join( 629 shlex.quote(str(arg)) for arg in check_trace.args 630 ) 631 _LOG.info( 632 '%s %s', 633 _COLOR.blue('Run ==>'), 634 quoted_command_args, 635 ) 636 637 638def apply_exclusions( 639 ctx: PresubmitContext, 640 paths: Sequence[Path] | None = None, 641) -> tuple[Path, ...]: 642 return ctx.format_options.filter_paths(paths or ctx.paths) 643