1#!/usr/bin/env python3 2# Copyright 2019 The Pigweed Authors 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); you may not 5# use this file except in compliance with the License. You may obtain a copy of 6# the License at 7# 8# https://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 13# License for the specific language governing permissions and limitations under 14# the License. 15"""Checks if the environment is set up correctly for Pigweed.""" 16 17import argparse 18from concurrent import futures 19import logging 20import json 21import os 22import pathlib 23import shutil 24import subprocess 25import sys 26import tempfile 27from typing import Callable, Iterable, Set 28 29import pw_cli.pw_command_plugins 30import pw_env_setup.cipd_setup.update as cipd_update 31from pw_env_setup import config_file 32 33 34def call_stdout(*args, **kwargs): 35 kwargs.update(stdout=subprocess.PIPE) 36 proc = subprocess.run(*args, **kwargs) 37 return proc.stdout.decode('utf-8') 38 39 40class _Fatal(Exception): 41 pass 42 43 44class Doctor: 45 def __init__( 46 self, *, log: logging.Logger | None = None, strict: bool = False 47 ): 48 self.strict = strict 49 self.log = log or logging.getLogger(__name__) 50 self.failures: Set[str] = set() 51 52 def run(self, checks: Iterable[Callable]): 53 with futures.ThreadPoolExecutor() as executor: 54 futures.wait( 55 [executor.submit(self._run_check, c, executor) for c in checks] 56 ) 57 58 def _run_check(self, check, executor): 59 ctx = DoctorContext(self, check.__name__, executor) 60 try: 61 self.log.debug('Running check %s', ctx.check) 62 check(ctx) 63 ctx.wait() 64 except _Fatal: 65 pass 66 except: # pylint: disable=bare-except 67 self.failures.add(ctx.check) 68 self.log.exception( 69 '%s failed with an unexpected exception', check.__name__ 70 ) 71 72 self.log.debug('Completed check %s', ctx.check) 73 74 75class DoctorContext: 76 """The context object provided to each context function.""" 77 78 def __init__(self, doctor: Doctor, check: str, executor: futures.Executor): 79 self._doctor = doctor 80 self.check = check 81 self._executor = executor 82 self._futures: list[futures.Future] = [] 83 84 def submit(self, function, *args, **kwargs): 85 """Starts running the provided function in parallel.""" 86 self._futures.append( 87 self._executor.submit(self._run_job, function, *args, **kwargs) 88 ) 89 90 def wait(self): 91 """Waits for all parallel tasks started with submit() to complete.""" 92 futures.wait(self._futures) 93 self._futures.clear() 94 95 def _run_job(self, function, *args, **kwargs): 96 try: 97 function(*args, **kwargs) 98 except _Fatal: 99 pass 100 except: # pylint: disable=bare-except 101 self._doctor.failures.add(self.check) 102 self._doctor.log.exception( 103 '%s failed with an unexpected exception', self.check 104 ) 105 106 def fatal(self, fmt, *args, **kwargs): 107 """Same as error() but terminates the check early.""" 108 self.error(fmt, *args, **kwargs) 109 raise _Fatal() 110 111 def error(self, fmt, *args, **kwargs): 112 self._doctor.log.error(fmt, *args, **kwargs) 113 self._doctor.failures.add(self.check) 114 115 def warning(self, fmt, *args, **kwargs): 116 if self._doctor.strict: 117 self.error(fmt, *args, **kwargs) 118 else: 119 self._doctor.log.warning(fmt, *args, **kwargs) 120 121 def info(self, fmt, *args, **kwargs): 122 self._doctor.log.info(fmt, *args, **kwargs) 123 124 def debug(self, fmt, *args, **kwargs): 125 self._doctor.log.debug(fmt, *args, **kwargs) 126 127 128def register_into(dest): 129 def decorate(func): 130 dest.append(func) 131 return func 132 133 return decorate 134 135 136CHECKS: list[Callable] = [] 137 138 139@register_into(CHECKS) 140def pw_plugins(ctx: DoctorContext): 141 if pw_cli.pw_command_plugins.errors(): 142 ctx.error('Not all pw plugins loaded successfully') 143 144 145def unames_are_equivalent( 146 uname_actual: str, uname_expected: str, rosetta: bool = False 147) -> bool: 148 """Determine if uname values are equivalent for this tool's purposes.""" 149 150 # Support `mac-arm64` through Rosetta until `mac-arm64` binaries are ready 151 # Expected and actual unames will not literally match on M1 Macs because 152 # they pretend to be Intel Macs for the purpose of environment setup. But 153 # that's intentional and doesn't require any user action. 154 if rosetta and "Darwin" in uname_expected and "arm64" in uname_expected: 155 uname_expected = uname_expected.replace("arm64", "x86_64") 156 157 return uname_actual == uname_expected 158 159 160@register_into(CHECKS) 161def env_os(ctx: DoctorContext): 162 """Check that the environment matches this machine.""" 163 if '_PW_ACTUAL_ENVIRONMENT_ROOT' not in os.environ: 164 return 165 env_root = pathlib.Path(os.environ['_PW_ACTUAL_ENVIRONMENT_ROOT']) 166 config = env_root / 'config.json' 167 if not config.is_file(): 168 return 169 170 with open(config, 'r') as ins: 171 data = json.load(ins) 172 if data['os'] != os.name: 173 ctx.error( 174 'Current OS (%s) does not match bootstrapped OS (%s)', 175 os.name, 176 data['os'], 177 ) 178 179 # Skipping sysname and nodename in os.uname(). nodename could change 180 # based on the current network. sysname won't change, but is 181 # redundant because it's contained in release or version, and 182 # skipping it here simplifies logic. 183 uname = ' '.join(getattr(os, 'uname', lambda: ())()[2:]) 184 rosetta_envvar = os.environ.get('_PW_ROSETTA', '0') 185 rosetta = rosetta_envvar.strip().lower() != '0' 186 if not unames_are_equivalent(uname, data['uname'], rosetta): 187 ctx.warning( 188 'Current uname (%s) does not match Bootstrap uname (%s), ' 189 'you may need to rerun bootstrap on this system', 190 uname, 191 data['uname'], 192 ) 193 194 195@register_into(CHECKS) 196def pw_root(ctx: DoctorContext): 197 """Check that environment variable PW_ROOT is set and makes sense.""" 198 try: 199 root = pathlib.Path(os.environ['PW_ROOT']).resolve() 200 except KeyError: 201 ctx.fatal('PW_ROOT not set') 202 203 # If pigweed is intentionally vendored and not in a git repo or submodule, 204 # set PW_DISABLE_ROOT_GIT_REPO_CHECK=1 during bootstrap to suppress the 205 # following check. 206 if os.environ.get('PW_DISABLE_ROOT_GIT_REPO_CHECK', '0') == '1': 207 return 208 209 git_root = pathlib.Path( 210 call_stdout(['git', 'rev-parse', '--show-toplevel'], cwd=root).strip() 211 ) 212 git_root = git_root.resolve() 213 if root != git_root: 214 if str(root).lower() != str(git_root).lower(): 215 ctx.error( 216 'PW_ROOT (%s) != `git rev-parse --show-toplevel` (%s)', 217 root, 218 git_root, 219 ) 220 else: 221 ctx.warning( 222 'PW_ROOT (%s) differs in case from ' 223 '`git rev-parse --show-toplevel` (%s)', 224 root, 225 git_root, 226 ) 227 228 229@register_into(CHECKS) 230def git_hook(ctx: DoctorContext): 231 """Check that presubmit git hook is installed.""" 232 if not os.environ.get('PW_ENABLE_PRESUBMIT_HOOK_WARNING'): 233 return 234 235 try: 236 root = pathlib.Path(os.environ['PW_ROOT']) 237 except KeyError: 238 return # This case is handled elsewhere. 239 240 hook = root / '.git' / 'hooks' / 'pre-push' 241 if not os.path.isfile(hook): 242 ctx.info( 243 'Presubmit hook not installed, please run ' 244 "'pw presubmit --install' before pushing changes." 245 ) 246 247 248@register_into(CHECKS) 249def python_version(ctx: DoctorContext): 250 """Check the Python version is correct.""" 251 actual = sys.version_info 252 expected = (3, 8) 253 if actual[0:2] < expected or actual[0] != expected[0]: 254 # If we get the wrong version but it still came from CIPD print a 255 # warning but give it a pass. 256 if 'chromium' in sys.version: 257 ctx.warning( 258 'Python %d.%d.x expected, got Python %d.%d.%d', 259 *expected, 260 *actual[0:3], 261 ) 262 else: 263 ctx.error( 264 'Python %d.%d.x required, got Python %d.%d.%d', 265 *expected, 266 *actual[0:3], 267 ) 268 269 270@register_into(CHECKS) 271def virtualenv(ctx: DoctorContext): 272 """Check that we're in the correct virtualenv.""" 273 try: 274 venv_path = pathlib.Path(os.environ['VIRTUAL_ENV']).resolve() 275 except KeyError: 276 ctx.error('VIRTUAL_ENV not set') 277 return 278 279 # When running in LUCI we might not have gone through the normal environment 280 # setup process, so we need to skip the rest of this step. 281 if 'LUCI_CONTEXT' in os.environ: 282 return 283 284 var = 'PW_ROOT' 285 if '_PW_ACTUAL_ENVIRONMENT_ROOT' in os.environ: 286 var = '_PW_ACTUAL_ENVIRONMENT_ROOT' 287 root = pathlib.Path(os.environ[var]).resolve() 288 289 if root not in venv_path.parents: 290 ctx.error('VIRTUAL_ENV (%s) not inside %s (%s)', venv_path, var, root) 291 ctx.error('\n'.join(os.environ.keys())) 292 293 294@register_into(CHECKS) 295def cipd(ctx: DoctorContext): 296 """Check cipd is set up correctly and in use.""" 297 if os.environ.get('PW_DOCTOR_SKIP_CIPD_CHECKS'): 298 return 299 300 cipd_path = 'pigweed' 301 302 cipd_exe = shutil.which('cipd') 303 if not cipd_exe: 304 ctx.fatal('cipd not in PATH (%s)', os.environ['PATH']) 305 306 temp = tempfile.NamedTemporaryFile(prefix='cipd', delete=False) 307 subprocess.run( 308 ['cipd', 'acl-check', '-json-output', temp.name, cipd_path], 309 stdout=subprocess.PIPE, 310 ) 311 if not json.load(temp)['result']: 312 ctx.fatal( 313 "can't access %s CIPD directory, have you run " 314 "'cipd auth-login'?", 315 cipd_path, 316 ) 317 318 commands_expected_from_cipd = [ 319 'arm-none-eabi-gcc', 320 'gn', 321 'ninja', 322 'protoc', 323 ] 324 325 # TODO(mohrr) get these tools in CIPD for Windows. 326 if os.name == 'posix': 327 commands_expected_from_cipd += [ 328 'clang++', 329 'openocd', 330 ] 331 332 for command in commands_expected_from_cipd: 333 path = shutil.which(command) 334 if path is None: 335 ctx.error( 336 'could not find %s in PATH (%s)', command, os.environ['PATH'] 337 ) 338 elif 'cipd' not in path: 339 ctx.warning( 340 'not using %s from cipd, got %s (path is %s)', 341 command, 342 path, 343 os.environ['PATH'], 344 ) 345 346 347@register_into(CHECKS) 348def cipd_versions(ctx: DoctorContext): 349 """Check cipd tool versions are current.""" 350 351 if os.environ.get('PW_DOCTOR_SKIP_CIPD_CHECKS'): 352 return 353 354 if 'PW_CIPD_INSTALL_DIR' not in os.environ: 355 ctx.error('PW_CIPD_INSTALL_DIR not set') 356 cipd_dir = pathlib.Path(os.environ['PW_CIPD_INSTALL_DIR']) 357 358 with open(cipd_dir / '_all_package_files.json', 'r') as ins: 359 json_paths = [pathlib.Path(x) for x in json.load(ins)] 360 361 platform = cipd_update.platform() 362 363 def check_cipd(package, install_path): 364 if platform not in package['platforms']: 365 ctx.debug( 366 "skipping %s because it doesn't apply to %s", 367 package['path'], 368 platform, 369 ) 370 return 371 372 tags_without_refs = [x for x in package['tags'] if ':' in x] 373 if not tags_without_refs: 374 ctx.debug( 375 'skipping %s because it tracks a ref, not a tag (%s)', 376 package['path'], 377 ', '.join(package['tags']), 378 ) 379 return 380 381 ctx.debug('checking version of %s', package['path']) 382 383 name = [part for part in package['path'].split('/') if '{' not in part][ 384 -1 385 ] 386 387 # If the exact path is specified in the JSON file use it, and require it 388 # exist. 389 if 'version_file' in package: 390 path = install_path / package['version_file'] 391 notify_method = ctx.error 392 # Otherwise, follow a heuristic to find the file but don't require the 393 # file to exist. 394 else: 395 path = install_path / '.versions' / f'{name}.cipd_version' 396 notify_method = ctx.debug 397 398 # Check if a .exe cipd_version exists on Windows. 399 path_windows = install_path / '.versions' / f'{name}.exe.cipd_version' 400 if os.name == 'nt' and path_windows.is_file(): 401 path = path_windows 402 403 if not path.is_file(): 404 notify_method(f'no version file for {name} at {path}') 405 return 406 407 with path.open() as ins: 408 installed = json.load(ins) 409 ctx.debug(f'found version file for {name} at {path}') 410 411 describe = ( 412 'cipd', 413 'describe', 414 installed['package_name'], 415 '-version', 416 installed['instance_id'], 417 ) 418 ctx.debug('%s', ' '.join(describe)) 419 output_raw = subprocess.check_output(describe).decode() 420 ctx.debug('output: %r', output_raw) 421 output = output_raw.split() 422 423 for tag in package['tags']: 424 if tag not in output: 425 ctx.error( 426 'CIPD package %s in %s is out of date, please rerun ' 427 'bootstrap', 428 installed['package_name'], 429 install_path, 430 ) 431 432 else: 433 ctx.debug( 434 'CIPD package %s in %s is current', 435 installed['package_name'], 436 install_path, 437 ) 438 439 deduped_packages = cipd_update.deduplicate_packages( 440 cipd_update.all_packages(json_paths) 441 ) 442 for json_path in json_paths: 443 ctx.debug(f'Checking packages in {json_path}') 444 if not json_path.exists(): 445 ctx.error( 446 'CIPD package file %s may have been deleted, please ' 447 'rerun bootstrap', 448 json_path, 449 ) 450 continue 451 452 install_path = pathlib.Path( 453 cipd_update.package_installation_path(cipd_dir, json_path) 454 ) 455 for package in json.loads(json_path.read_text()).get('packages', ()): 456 # Ensure package matches deduped_packages format 457 cipd_update.update_subdir(package, json_path) 458 if package not in deduped_packages: 459 ctx.debug( 460 f'Skipping overridden package {package["path"]} ' 461 f'with tag(s) {package["tags"]}' 462 ) 463 continue 464 ctx.submit(check_cipd, package, install_path) 465 466 467@register_into(CHECKS) 468def symlinks(ctx: DoctorContext): 469 """Check that the platform supports symlinks.""" 470 471 try: 472 root = pathlib.Path(os.environ['PW_ROOT']).resolve() 473 except KeyError: 474 return # This case is handled elsewhere. 475 476 with tempfile.TemporaryDirectory() as tmpdir: 477 dest = pathlib.Path(tmpdir).resolve() / 'symlink' 478 try: 479 os.symlink(root, dest) 480 failure = False 481 except OSError: 482 # TODO(pwbug/500) Find out what errno is set when symlinks aren't 483 # supported by the OS. 484 failure = True 485 486 if not os.path.islink(dest) or failure: 487 ctx.warning( 488 'Symlinks are not supported or current user does not have ' 489 'permission to use them. This may cause build issues. If on ' 490 'Windows, turn on Development Mode to enable symlink support.' 491 ) 492 493 494def run_doctor(strict=False, checks=None): 495 """Run all the Check subclasses defined in this file.""" 496 497 config = config_file.load().get('pw', {}).get('pw_doctor', {}) 498 new_bug_url = config.get('new_bug_url', 'https://issues.pigweed.dev/new') 499 500 if checks is None: 501 checks = tuple(CHECKS) 502 503 doctor = Doctor(strict=strict) 504 doctor.log.debug('Doctor running %d checks...', len(checks)) 505 506 doctor.run(checks) 507 508 if doctor.failures: 509 doctor.log.info('Failed checks: %s', ', '.join(doctor.failures)) 510 doctor.log.info( 511 "Your environment setup has completed, but something isn't right " 512 'and some things may not work correctly. You may continue with ' 513 'development, but please seek support at %s or by ' 514 'reaching out to your team.', 515 new_bug_url, 516 ) 517 else: 518 doctor.log.info('Environment passes all checks!') 519 return len(doctor.failures) 520 521 522def main() -> int: 523 """Check that the environment is set up correctly for Pigweed.""" 524 parser = argparse.ArgumentParser(description=__doc__) 525 parser.add_argument( 526 '--strict', 527 action='store_true', 528 help='Run additional checks.', 529 ) 530 531 return run_doctor(**vars(parser.parse_args())) 532 533 534if __name__ == '__main__': 535 # By default, display log messages like a simple print statement. 536 logging.basicConfig(format='%(message)s', level=logging.INFO) 537 sys.exit(main()) 538