1#!/usr/bin/env python3 2# Copyright 2020 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6"""Get an upstream patch to LLVM's PATCHES.json.""" 7 8import argparse 9import dataclasses 10import datetime 11import json 12import logging 13import os 14from pathlib import Path 15import subprocess 16import sys 17import typing as t 18 19import chroot 20import get_llvm_hash 21import git 22import git_llvm_rev 23import patch_utils 24 25 26__DOC_EPILOGUE = """ 27Example Usage: 28 get_upstream_patch --chromeos_path ~/chromiumos --platform chromiumos \ 29--sha 1234567 --sha 890abdc 30""" 31 32 33class CherrypickError(ValueError): 34 """A ValueError that highlights the cherry-pick has been seen before""" 35 36 37class CherrypickVersionError(ValueError): 38 """A ValueError that highlights the cherry-pick is before the start_sha""" 39 40 41class PatchApplicationError(ValueError): 42 """A ValueError indicating that a test patch application was unsuccessful""" 43 44 45def validate_patch_application( 46 llvm_dir: Path, svn_version: int, patches_json_fp: Path, patch_props 47): 48 start_sha = get_llvm_hash.GetGitHashFrom(llvm_dir, svn_version) 49 subprocess.run(["git", "-C", llvm_dir, "checkout", start_sha], check=True) 50 51 predecessor_apply_results = patch_utils.apply_all_from_json( 52 svn_version, llvm_dir, patches_json_fp, continue_on_failure=True 53 ) 54 55 if predecessor_apply_results.failed_patches: 56 logging.error("Failed to apply patches from PATCHES.json:") 57 for p in predecessor_apply_results.failed_patches: 58 logging.error("Patch title: %s", p.title()) 59 raise PatchApplicationError("Failed to apply patch from PATCHES.json") 60 61 patch_entry = patch_utils.PatchEntry.from_dict( 62 patches_json_fp.parent, patch_props 63 ) 64 test_apply_result = patch_entry.test_apply(Path(llvm_dir)) 65 66 if not test_apply_result: 67 logging.error("Could not apply requested patch") 68 logging.error(test_apply_result.failure_info()) 69 raise PatchApplicationError( 70 f'Failed to apply patch: {patch_props["metadata"]["title"]}' 71 ) 72 73 74def add_patch( 75 patches_json_path: str, 76 patches_dir: str, 77 relative_patches_dir: str, 78 start_version: git_llvm_rev.Rev, 79 llvm_dir: t.Union[Path, str], 80 rev: t.Union[git_llvm_rev.Rev, str], 81 sha: str, 82 package: str, 83 platforms: t.Iterable[str], 84): 85 """Gets the start and end intervals in 'json_file'. 86 87 Args: 88 patches_json_path: The absolute path to PATCHES.json. 89 patches_dir: The aboslute path to the directory patches are in. 90 relative_patches_dir: The relative path to PATCHES.json. 91 start_version: The base LLVM revision this patch applies to. 92 llvm_dir: The path to LLVM checkout. 93 rev: An LLVM revision (git_llvm_rev.Rev) for a cherrypicking, or a 94 differential revision (str) otherwise. 95 sha: The LLVM git sha that corresponds to the patch. For differential 96 revisions, the git sha from the local commit created by 'arc patch' 97 is used. 98 package: The LLVM project name this patch applies to. 99 platforms: List of platforms this patch applies to. 100 101 Raises: 102 CherrypickError: A ValueError that highlights the cherry-pick has been 103 seen before. 104 CherrypickRangeError: A ValueError that's raised when the given patch 105 is from before the start_sha. 106 """ 107 108 is_cherrypick = isinstance(rev, git_llvm_rev.Rev) 109 if is_cherrypick: 110 file_name = f"{sha}.patch" 111 else: 112 file_name = f"{rev}.patch" 113 rel_patch_path = os.path.join(relative_patches_dir, file_name) 114 115 # Check that we haven't grabbed a patch range that's nonsensical. 116 end_vers = rev.number if isinstance(rev, git_llvm_rev.Rev) else None 117 if end_vers is not None and end_vers <= start_version.number: 118 raise CherrypickVersionError( 119 f"`until` version {end_vers} is earlier or equal to" 120 f" `from` version {start_version.number} for patch" 121 f" {rel_patch_path}" 122 ) 123 124 with open(patches_json_path, encoding="utf-8") as f: 125 contents = f.read() 126 indent_len = patch_utils.predict_indent(contents.splitlines()) 127 patches_json = json.loads(contents) 128 129 for p in patches_json: 130 rel_path = p["rel_patch_path"] 131 if rel_path == rel_patch_path: 132 raise CherrypickError( 133 f"Patch at {rel_path} already exists in PATCHES.json" 134 ) 135 if is_cherrypick: 136 if sha in rel_path: 137 logging.warning( 138 "Similarly-named patch already exists in PATCHES.json: %r", 139 rel_path, 140 ) 141 142 with open(os.path.join(patches_dir, file_name), "wb") as f: 143 cmd = ["git", "show", sha] 144 # Only apply the part of the patch that belongs to this package, expect 145 # LLVM. This is because some packages are built with LLVM ebuild on X86 146 # but not on the other architectures. e.g. compiler-rt. Therefore 147 # always apply the entire patch to LLVM ebuild as a workaround. 148 if package != "llvm": 149 cmd.append(package_to_project(package)) 150 subprocess.check_call(cmd, stdout=f, cwd=llvm_dir) 151 152 commit_subject = subprocess.check_output( 153 ["git", "log", "-n1", "--format=%s", sha], 154 cwd=llvm_dir, 155 encoding="utf-8", 156 ) 157 patch_props = { 158 "rel_patch_path": rel_patch_path, 159 "metadata": { 160 "title": commit_subject.strip(), 161 "info": [], 162 }, 163 "platforms": sorted(platforms), 164 "version_range": { 165 "from": start_version.number, 166 "until": end_vers, 167 }, 168 } 169 170 with patch_utils.git_clean_context(Path(llvm_dir)): 171 validate_patch_application( 172 Path(llvm_dir), 173 start_version.number, 174 Path(patches_json_path), 175 patch_props, 176 ) 177 178 patches_json.append(patch_props) 179 180 temp_file = patches_json_path + ".tmp" 181 with open(temp_file, "w", encoding="utf-8") as f: 182 json.dump( 183 patches_json, 184 f, 185 indent=indent_len, 186 separators=(",", ": "), 187 sort_keys=True, 188 ) 189 f.write("\n") 190 os.rename(temp_file, patches_json_path) 191 192 193# Resolves a git ref (or similar) to a LLVM SHA. 194def resolve_llvm_ref(llvm_dir: t.Union[Path, str], sha: str) -> str: 195 return subprocess.check_output( 196 ["git", "rev-parse", sha], 197 encoding="utf-8", 198 cwd=llvm_dir, 199 ).strip() 200 201 202# Get the package name of an LLVM project 203def project_to_package(project: str) -> str: 204 if project == "libunwind": 205 return "llvm-libunwind" 206 return project 207 208 209# Get the LLVM project name of a package 210def package_to_project(package: str) -> str: 211 if package == "llvm-libunwind": 212 return "libunwind" 213 return package 214 215 216# Get the LLVM projects change in the specifed sha 217def get_package_names(sha: str, llvm_dir: t.Union[Path, str]) -> list: 218 paths = subprocess.check_output( 219 ["git", "show", "--name-only", "--format=", sha], 220 cwd=llvm_dir, 221 encoding="utf-8", 222 ).splitlines() 223 # Some LLVM projects are built by LLVM ebuild on X86, so always apply the 224 # patch to LLVM ebuild 225 packages = {"llvm"} 226 # Detect if there are more packages to apply the patch to 227 for path in paths: 228 package = project_to_package(path.split("/")[0]) 229 if package in ("compiler-rt", "libcxx", "libcxxabi", "llvm-libunwind"): 230 packages.add(package) 231 return list(sorted(packages)) 232 233 234def create_patch_for_packages( 235 packages: t.List[str], 236 symlinks: t.List[str], 237 start_rev: git_llvm_rev.Rev, 238 rev: t.Union[git_llvm_rev.Rev, str], 239 sha: str, 240 llvm_dir: t.Union[Path, str], 241 platforms: t.Iterable[str], 242): 243 """Create a patch and add its metadata for each package""" 244 for package, symlink in zip(packages, symlinks): 245 symlink_dir = os.path.dirname(symlink) 246 patches_json_path = os.path.join(symlink_dir, "files/PATCHES.json") 247 relative_patches_dir = "cherry" if package == "llvm" else "" 248 patches_dir = os.path.join(symlink_dir, "files", relative_patches_dir) 249 logging.info("Getting %s (%s) into %s", rev, sha, package) 250 add_patch( 251 patches_json_path, 252 patches_dir, 253 relative_patches_dir, 254 start_rev, 255 llvm_dir, 256 rev, 257 sha, 258 package, 259 platforms=platforms, 260 ) 261 262 263def make_cl( 264 llvm_symlink_dir: str, 265 branch: str, 266 commit_messages: t.List[str], 267 reviewers: t.Optional[t.List[str]], 268 cc: t.Optional[t.List[str]], 269): 270 subprocess.check_output(["git", "add", "--all"], cwd=llvm_symlink_dir) 271 git.CommitChanges(llvm_symlink_dir, commit_messages) 272 git.UploadChanges(llvm_symlink_dir, branch, reviewers, cc) 273 git.DeleteBranch(llvm_symlink_dir, branch) 274 275 276def resolve_symbolic_sha(start_sha: str, chromeos_path: Path) -> str: 277 if start_sha == "llvm": 278 return get_llvm_hash.LLVMHash().GetCrOSCurrentLLVMHash(chromeos_path) 279 280 if start_sha == "llvm-next": 281 return get_llvm_hash.LLVMHash().GetCrOSLLVMNextHash() 282 283 return start_sha 284 285 286def find_patches_and_make_cl( 287 chromeos_path: str, 288 patches: t.List[str], 289 start_rev: git_llvm_rev.Rev, 290 llvm_config: git_llvm_rev.LLVMConfig, 291 llvm_symlink_dir: str, 292 allow_failures: bool, 293 create_cl: bool, 294 skip_dependencies: bool, 295 reviewers: t.Optional[t.List[str]], 296 cc: t.Optional[t.List[str]], 297 platforms: t.Iterable[str], 298): 299 converted_patches = [ 300 _convert_patch(llvm_config, skip_dependencies, p) for p in patches 301 ] 302 potential_duplicates = _get_duplicate_shas(converted_patches) 303 if potential_duplicates: 304 err_msg = "\n".join( 305 f"{a.patch} == {b.patch}" for a, b in potential_duplicates 306 ) 307 raise RuntimeError(f"Found Duplicate SHAs:\n{err_msg}") 308 309 # CL Related variables, only used if `create_cl` 310 commit_messages = [ 311 "llvm: get patches from upstream\n", 312 ] 313 branch = ( 314 f'get-upstream-{datetime.datetime.now().strftime("%Y%m%d%H%M%S%f")}' 315 ) 316 317 if create_cl: 318 git.CreateBranch(llvm_symlink_dir, branch) 319 320 successes = [] 321 failures = [] 322 for parsed_patch in converted_patches: 323 # Find out the llvm projects changed in this commit 324 packages = get_package_names(parsed_patch.sha, llvm_config.dir) 325 # Find out the ebuild of the corresponding ChromeOS packages 326 ebuild_paths = chroot.GetChrootEbuildPaths( 327 chromeos_path, 328 [ 329 "sys-devel/llvm" if package == "llvm" else "sys-libs/" + package 330 for package in packages 331 ], 332 ) 333 ebuild_paths = chroot.ConvertChrootPathsToAbsolutePaths( 334 chromeos_path, ebuild_paths 335 ) 336 # Create a local patch for all the affected llvm projects 337 try: 338 create_patch_for_packages( 339 packages, 340 ebuild_paths, 341 start_rev, 342 parsed_patch.rev, 343 parsed_patch.sha, 344 llvm_config.dir, 345 platforms=platforms, 346 ) 347 except PatchApplicationError as e: 348 if allow_failures: 349 logging.warning(e) 350 failures.append(parsed_patch.sha) 351 continue 352 else: 353 raise e 354 successes.append(parsed_patch.sha) 355 356 if create_cl: 357 commit_messages.extend( 358 [ 359 parsed_patch.git_msg(), 360 subprocess.check_output( 361 ["git", "log", "-n1", "--oneline", parsed_patch.sha], 362 cwd=llvm_config.dir, 363 encoding="utf-8", 364 ), 365 ] 366 ) 367 368 if parsed_patch.is_differential: 369 subprocess.check_output( 370 ["git", "reset", "--hard", "HEAD^"], cwd=llvm_config.dir 371 ) 372 373 if allow_failures: 374 success_list = (":\n\t" + "\n\t".join(successes)) if successes else "." 375 logging.info( 376 "Successfully applied %d patches%s", len(successes), success_list 377 ) 378 failure_list = (":\n\t" + "\n\t".join(failures)) if failures else "." 379 logging.info( 380 "Failed to apply %d patches%s", len(failures), failure_list 381 ) 382 383 if successes and create_cl: 384 make_cl( 385 llvm_symlink_dir, 386 branch, 387 commit_messages, 388 reviewers, 389 cc, 390 ) 391 392 393@dataclasses.dataclass(frozen=True) 394class ParsedPatch: 395 """Class to keep track of bundled patch info.""" 396 397 patch: str 398 sha: str 399 is_differential: bool 400 rev: t.Union[git_llvm_rev.Rev, str] 401 402 def git_msg(self) -> str: 403 if self.is_differential: 404 return f"\n\nreviews.llvm.org/{self.patch}\n" 405 return f"\n\nreviews.llvm.org/rG{self.sha}\n" 406 407 408def _convert_patch( 409 llvm_config: git_llvm_rev.LLVMConfig, skip_dependencies: bool, patch: str 410) -> ParsedPatch: 411 """Extract git revision info from a patch. 412 413 Args: 414 llvm_config: LLVM configuration object. 415 skip_dependencies: Pass --skip-dependecies for to `arc` 416 patch: A single patch referent string. 417 418 Returns: 419 A [ParsedPatch] object. 420 """ 421 422 # git hash should only have lower-case letters 423 is_differential = patch.startswith("D") 424 if is_differential: 425 subprocess.check_output( 426 [ 427 "arc", 428 "patch", 429 "--nobranch", 430 "--skip-dependencies" if skip_dependencies else "--revision", 431 patch, 432 ], 433 cwd=llvm_config.dir, 434 ) 435 sha = resolve_llvm_ref(llvm_config.dir, "HEAD") 436 rev: t.Union[git_llvm_rev.Rev, str] = patch 437 else: 438 sha = resolve_llvm_ref(llvm_config.dir, patch) 439 rev = git_llvm_rev.translate_sha_to_rev(llvm_config, sha) 440 return ParsedPatch( 441 patch=patch, sha=sha, rev=rev, is_differential=is_differential 442 ) 443 444 445def _get_duplicate_shas( 446 patches: t.List[ParsedPatch], 447) -> t.List[t.Tuple[ParsedPatch, ParsedPatch]]: 448 """Return a list of Patches which have duplicate SHA's""" 449 return [ 450 (left, right) 451 for i, left in enumerate(patches) 452 for right in patches[i + 1 :] 453 if left.sha == right.sha 454 ] 455 456 457def get_from_upstream( 458 chromeos_path: str, 459 create_cl: bool, 460 start_sha: str, 461 patches: t.List[str], 462 platforms: t.Iterable[str], 463 allow_failures: bool = False, 464 skip_dependencies: bool = False, 465 reviewers: t.Optional[t.List[str]] = None, 466 cc: t.Optional[t.List[str]] = None, 467): 468 llvm_symlink = chroot.ConvertChrootPathsToAbsolutePaths( 469 chromeos_path, 470 chroot.GetChrootEbuildPaths(chromeos_path, ["sys-devel/llvm"]), 471 )[0] 472 llvm_symlink_dir = os.path.dirname(llvm_symlink) 473 474 git_status = subprocess.check_output( 475 ["git", "status", "-s"], cwd=llvm_symlink_dir, encoding="utf-8" 476 ) 477 478 if git_status: 479 error_path = os.path.dirname(os.path.dirname(llvm_symlink_dir)) 480 raise ValueError(f"Uncommited changes detected in {error_path}") 481 482 start_sha = resolve_symbolic_sha(start_sha, Path(chromeos_path)) 483 logging.info("Base llvm hash == %s", start_sha) 484 485 llvm_config = git_llvm_rev.LLVMConfig( 486 remote="origin", dir=get_llvm_hash.GetAndUpdateLLVMProjectInLLVMTools() 487 ) 488 start_sha = resolve_llvm_ref(llvm_config.dir, start_sha) 489 490 find_patches_and_make_cl( 491 chromeos_path=chromeos_path, 492 patches=patches, 493 platforms=platforms, 494 start_rev=git_llvm_rev.translate_sha_to_rev(llvm_config, start_sha), 495 llvm_config=llvm_config, 496 llvm_symlink_dir=llvm_symlink_dir, 497 create_cl=create_cl, 498 skip_dependencies=skip_dependencies, 499 reviewers=reviewers, 500 cc=cc, 501 allow_failures=allow_failures, 502 ) 503 504 logging.info("Complete.") 505 506 507def main(): 508 chroot.VerifyOutsideChroot() 509 logging.basicConfig( 510 format="%(asctime)s: %(levelname)s: %(filename)s:%(lineno)d: " 511 "%(message)s", 512 level=logging.INFO, 513 ) 514 515 parser = argparse.ArgumentParser( 516 description=__doc__, 517 formatter_class=argparse.RawDescriptionHelpFormatter, 518 epilog=__DOC_EPILOGUE, 519 ) 520 parser.add_argument( 521 "--chromeos_path", 522 default=os.path.join(os.path.expanduser("~"), "chromiumos"), 523 help="the path to the chroot (default: %(default)s)", 524 ) 525 parser.add_argument( 526 "--start_sha", 527 default="llvm-next", 528 help="LLVM SHA that the patch should start applying at. You can " 529 'specify "llvm" or "llvm-next", as well. Defaults to %(default)s.', 530 ) 531 parser.add_argument( 532 "--sha", 533 action="append", 534 default=[], 535 help="The LLVM git SHA to cherry-pick.", 536 ) 537 parser.add_argument( 538 "--differential", 539 action="append", 540 default=[], 541 help="The LLVM differential revision to apply. Example: D1234." 542 " Cannot be used for changes already merged upstream; use --sha" 543 " instead for those.", 544 ) 545 parser.add_argument( 546 "--platform", 547 action="append", 548 required=True, 549 help="Apply this patch to the give platform. Common options include " 550 '"chromiumos" and "android". Can be specified multiple times to ' 551 "apply to multiple platforms", 552 ) 553 parser.add_argument( 554 "--allow_failures", 555 action="store_true", 556 help="Skip patches that fail to apply and continue.", 557 ) 558 parser.add_argument( 559 "--create_cl", 560 action="store_true", 561 help="Automatically create a CL if specified", 562 ) 563 parser.add_argument( 564 "--skip_dependencies", 565 action="store_true", 566 help="Skips a LLVM differential revision's dependencies. Only valid " 567 "when --differential appears exactly once.", 568 ) 569 args = parser.parse_args() 570 chroot.VerifyChromeOSRoot(args.chromeos_path) 571 572 if not (args.sha or args.differential): 573 parser.error("--sha or --differential required") 574 575 if args.skip_dependencies and len(args.differential) != 1: 576 parser.error( 577 "--skip_dependencies is only valid when there's exactly one " 578 "supplied differential" 579 ) 580 581 get_from_upstream( 582 chromeos_path=args.chromeos_path, 583 allow_failures=args.allow_failures, 584 create_cl=args.create_cl, 585 start_sha=args.start_sha, 586 patches=args.sha + args.differential, 587 skip_dependencies=args.skip_dependencies, 588 platforms=args.platform, 589 ) 590 591 592if __name__ == "__main__": 593 sys.exit(main()) 594