1#!/usr/bin/env -S python3 -B 2# 3# Copyright (C) 2020 The Android Open Source Project 4# 5# Licensed under the Apache License, Version 2.0 (the "License"); 6# you may not use this file except in compliance with the License. 7# You may obtain a copy of the License at 8# 9# http://www.apache.org/licenses/LICENSE-2.0 10# 11# Unless required by applicable law or agreed to in writing, software 12# distributed under the License is distributed on an "AS IS" BASIS, 13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14# See the License for the specific language governing permissions and 15# limitations under the License. 16 17"""Downloads prebuilts for ART Module dependencies and creates CLs in git.""" 18 19import argparse 20import collections 21import os 22import subprocess 23import sys 24import tempfile 25 26 27# Prebuilt description used in commit message 28PREBUILT_DESCR = "ART dependencies" 29 30 31# Branch and target tuple for a CI build 32BuildSource = collections.namedtuple("BuildSource", [ 33 "branch", 34 "target", 35]) 36 37 38# The list of arches does not include riscv64: it is not supported by mainline, 39# and it is updated from a local build with --local-dist-riscv64 option (which 40# overwrites the list of arches to include only riscv64). 41# 42# TODO(b/286551985): add riscv64 here and don't override the global list. 43ARCHES = ["arm", "arm64", "x86", "x86_64"] 44 45# CI build for SDK snapshots 46SDK_SOURCE = BuildSource("aosp-main", 47 "mainline_modules_sdks-trunk_staging-userdebug") 48 49# Architecture-specific CI builds for APEXes. These are only used in the chroot 50# test setup (see art/tools/buildbot-build.sh). They never become part of any 51# dist artifact. 52# 53# There are currently no CI builds except for x86_64, so APEX updates are 54# skipped by default. 55APEX_SOURCE = { 56 "x86_64": BuildSource("aosp-main", 57 "mainline_modules_x86_64-trunk_staging-userdebug"), 58} 59 60# Paths to git projects to prepare CLs in 61GIT_PROJECT_ROOTS = [ 62 "prebuilts/module_sdk/conscrypt", 63 "prebuilts/runtime", 64] 65 66SDK_VERSION = "current" 67 68SCRIPT_PATH = "prebuilts/runtime/mainline/update.py" 69 70 71InstallEntry = collections.namedtuple( 72 "InstallEntry", 73 [ 74 # Either "apex" or "module_sdk", for the --skip-* flags. 75 "type", 76 # Source CI build as a BuildSource tuple, or None if none exists. 77 "source_build", 78 # Artifact path in the build, passed to fetch_target. Should match a 79 # single file. "{BUILD}" gets replaced with the build number. 80 "source_path", 81 # Local install path. 82 "install_path", 83 # True if the entry is a zip file that should be unzipped to 84 # install_path. 85 "install_unzipped", 86 # If set, the entry is a zip from which this single file is extracted to 87 # install_path. 88 "unzip_single_file", 89 ], 90 defaults=(False, None), # Defaults for install_unzipped and unzip_single_file. 91) 92 93 94def install_apex_entries(module_name, apex_name): 95 return [ 96 InstallEntry( 97 type="apex", 98 source_build=APEX_SOURCE.get(arch, None), 99 source_path=os.path.join( 100 "mainline_modules_" + arch, 101 apex_name + ".apex"), 102 install_path=os.path.join( 103 "prebuilts/runtime/mainline", 104 module_name, 105 "apex", 106 apex_name + "-" + arch + ".apex")) 107 for arch in ARCHES] 108 109 110def install_unbundled_sdk_entries(apex_name, mainline_sdk_name, sdk_type, install_path): 111 if "riscv64" in ARCHES: 112 if sdk_type == "host-exports": # no host prebuilts for riscv64 113 return [] 114 install_path = os.path.join("prebuilts/runtime/mainline/local_riscv64", 115 install_path) 116 return [ 117 InstallEntry( 118 type="module_sdk", 119 source_build=SDK_SOURCE, 120 source_path=os.path.join( 121 "mainline-sdks/for-latest-build", 122 SDK_VERSION, 123 apex_name, 124 sdk_type, 125 mainline_sdk_name + "-" + sdk_type + "-" + SDK_VERSION + ".zip"), 126 install_path=install_path, 127 install_unzipped=True)] 128 129 130def install_bundled_sdk_entries(module_name, sdk_type): 131 if "riscv64" in ARCHES: 132 if sdk_type == "host-exports": # no host prebuilts for riscv64 133 return [] 134 return [ 135 InstallEntry( 136 type="module_sdk", 137 source_build=SDK_SOURCE, 138 source_path=os.path.join( 139 "bundled-mainline-sdks", 140 "com.android." + module_name, 141 sdk_type, 142 module_name + "-module-" + sdk_type + "-" + SDK_VERSION + ".zip"), 143 install_path=os.path.join( 144 "prebuilts/runtime/mainline", 145 module_name, 146 sdk_type), 147 install_unzipped=True)] 148 149 150def install_platform_mainline_sdk_entries(sdk_type): 151 if "riscv64" in ARCHES: 152 if sdk_type == "host-exports": # no host prebuilts for riscv64 153 return [] 154 return [ 155 InstallEntry( 156 type="module_sdk", 157 source_build=SDK_SOURCE, 158 source_path=os.path.join( 159 "bundled-mainline-sdks", 160 "platform-mainline", 161 sdk_type, 162 "platform-mainline-" + sdk_type + "-" + SDK_VERSION + ".zip"), 163 install_path=os.path.join( 164 "prebuilts/runtime/mainline/platform", 165 sdk_type), 166 install_unzipped=True)] 167 168 169# This is defined as a function (not a global list) because it depends on the 170# list of architectures, which may change after parsing options. 171def install_entries(): 172 return ( 173 # Conscrypt 174 install_apex_entries("conscrypt", "com.android.conscrypt") + 175 install_unbundled_sdk_entries( 176 "com.android.conscrypt", "conscrypt-module", "sdk", 177 "prebuilts/module_sdk/conscrypt/current") + 178 install_unbundled_sdk_entries( 179 "com.android.conscrypt", "conscrypt-module", "test-exports", 180 "prebuilts/module_sdk/conscrypt/current/test-exports") + 181 install_unbundled_sdk_entries( 182 "com.android.conscrypt", "conscrypt-module", "host-exports", 183 "prebuilts/module_sdk/conscrypt/current/host-exports") + 184 185 # Runtime (Bionic) 186 # sdk and host-exports must always be updated together, because the linker 187 # and the CRT object files gets embedded in the binaries on linux host 188 # Bionic (see code and comments around host_bionic_linker_script in 189 # build/soong). 190 install_apex_entries("runtime", "com.android.runtime") + 191 install_bundled_sdk_entries("runtime", "sdk") + 192 install_bundled_sdk_entries("runtime", "host-exports") + 193 194 # I18N 195 install_apex_entries("i18n", "com.android.i18n") + 196 install_bundled_sdk_entries("i18n", "sdk") + 197 install_bundled_sdk_entries("i18n", "test-exports") + 198 199 # tzdata 200 install_apex_entries("tzdata", "com.android.tzdata") + 201 install_bundled_sdk_entries("tzdata", "test-exports") + 202 203 # Platform 204 install_platform_mainline_sdk_entries("sdk") + 205 install_platform_mainline_sdk_entries("test-exports") + 206 207 []) 208 209 210def check_call(cmd, **kwargs): 211 """Proxy for subprocess.check_call with logging.""" 212 msg = " ".join(cmd) if isinstance(cmd, list) else cmd 213 if "cwd" in kwargs: 214 msg = "In " + kwargs["cwd"] + ": " + msg 215 print(msg) 216 subprocess.check_call(cmd, **kwargs) 217 218 219def fetch_artifact(branch, target, build, fetch_pattern, local_dir, zip_entry=None): 220 """Fetches artifact from the build server.""" 221 fetch_artifact_path = "/google/data/ro/projects/android/fetch_artifact" 222 cmd = [fetch_artifact_path, "--branch", branch, "--target", target, 223 "--bid", build, fetch_pattern] 224 if zip_entry: 225 cmd += ["--zip_entry", zip_entry] 226 check_call(cmd, cwd=local_dir) 227 228 229def start_branch(git_branch_name, git_dirs): 230 """Creates a new repo branch in the given projects.""" 231 check_call(["repo", "start", git_branch_name] + git_dirs) 232 # In case the branch already exists we reset it to upstream, to get a clean 233 # update CL. 234 for git_dir in git_dirs: 235 check_call(["git", "reset", "--hard", "@{upstream}"], cwd=git_dir) 236 237 238def upload_branch(git_root, git_branch_name): 239 """Uploads the CLs in the given branch in the given project.""" 240 # Set the branch as topic to bundle with the CLs in other git projects (if 241 # any). 242 check_call(["repo", "upload", "-t", "--br=" + git_branch_name, git_root]) 243 244 245def remove_files(git_root, subpaths, stage_removals): 246 """Removes files in the work tree, optionally staging them in git.""" 247 if stage_removals: 248 check_call(["git", "rm", "-qrf", "--ignore-unmatch"] + subpaths, cwd=git_root) 249 # Need a plain rm afterwards even if git rm was executed, because git won't 250 # remove directories if they have non-git files in them. 251 check_call(["rm", "-rf"] + subpaths, cwd=git_root) 252 253 254def commit(git_root, prebuilt_descr, installed_sources, add_paths, bug_number): 255 """Commits the new prebuilts.""" 256 check_call(["git", "add"] + 257 [path for path in add_paths 258 if os.path.exists(os.path.join(git_root, path))], 259 cwd=git_root) 260 261 if installed_sources: 262 message = "Update {} prebuilts.\n\n".format(prebuilt_descr) 263 if len(installed_sources) == 1: 264 message += "Taken from {}.".format(installed_sources[0]) 265 else: 266 message += "Taken from:\n{}".format( 267 "\n".join([s.capitalize() for s in installed_sources])) 268 else: 269 # For riscv64, update from a local tree is the only available option. 270 message = "" if "riscv64" in ARCHES else "DO NOT SUBMIT: " 271 message += ( 272 "Update {prebuilt_descr} prebuilts from local build." 273 .format(prebuilt_descr=prebuilt_descr)) 274 message += ("\n\nCL prepared by {}." 275 "\n\nTest: Presubmits".format(SCRIPT_PATH)) 276 if bug_number: 277 message += ("\nBug: {}".format(bug_number)) 278 279 msg_fd, msg_path = tempfile.mkstemp() 280 try: 281 with os.fdopen(msg_fd, "w") as f: 282 f.write(message) 283 # Do a diff first to skip the commit without error if there are no changes 284 # to commit. 285 check_call("git diff-index --quiet --cached HEAD -- || " 286 "git commit -F " + msg_path, shell=True, cwd=git_root) 287 finally: 288 os.unlink(msg_path) 289 290 291def install_entry(tmp_dir, local_dist, build_numbers, entry): 292 """Installs one file specified by entry.""" 293 294 if not local_dist and not entry.source_build: 295 print("WARNING: No CI build for {} - skipping.".format(entry.source_path)) 296 return None 297 298 build_number = (build_numbers[entry.source_build.branch] 299 if build_numbers else None) 300 301 # Fall back to the username as the build ID if we have no build number. That's 302 # what a dist install does in a local build. 303 source_path = entry.source_path.replace( 304 "{BUILD}", str(build_number) if build_number else os.getenv("USER")) 305 306 source_dir, source_file = os.path.split(source_path) 307 308 if local_dist: 309 download_dir = os.path.join(tmp_dir, source_dir) 310 else: 311 download_dir = os.path.join(tmp_dir, 312 entry.source_build.branch, 313 build_number, 314 entry.source_build.target, 315 source_dir) 316 os.makedirs(download_dir, exist_ok=True) 317 download_file = os.path.join(download_dir, source_file) 318 319 unzip_dir = None 320 unzip_file = None 321 if entry.unzip_single_file: 322 unzip_dir = os.path.join(download_dir, 323 source_path.removesuffix(".zip") + "_unzip") 324 os.makedirs(unzip_dir, exist_ok=True) 325 unzip_file = os.path.join(unzip_dir, entry.unzip_single_file) 326 327 if not local_dist and unzip_file: 328 if not os.path.exists(unzip_file): 329 # Use the --zip_entry option to fetch_artifact to avoid downloading the 330 # whole zip. 331 fetch_artifact(entry.source_build.branch, 332 entry.source_build.target, 333 build_number, 334 source_path, 335 unzip_dir, 336 entry.unzip_single_file) 337 if not os.path.exists(unzip_file): 338 sys.exit("fetch_artifact didn't create expected file {}".format(unzip_file)) 339 else: 340 # Fetch files once by downloading them into a specific location in tmp_dir 341 # only if they're not already there. 342 if not os.path.exists(download_file): 343 if local_dist: 344 check_call(["cp", os.path.join(local_dist, source_path), download_dir]) 345 else: 346 fetch_artifact(entry.source_build.branch, 347 entry.source_build.target, 348 build_number, 349 source_path, 350 download_dir) 351 if not os.path.exists(download_file): 352 sys.exit("Failed to retrieve {}".format(source_path)) 353 354 install_dir, install_file = os.path.split(entry.install_path) 355 os.makedirs(install_dir, exist_ok=True) 356 357 if entry.install_unzipped: 358 if "riscv64" in ARCHES: 359 tmp_dir = os.path.join(install_file, "tmp") 360 check_call(["mkdir", "-p", tmp_dir], cwd=install_dir) 361 check_call(["unzip", "-DD", "-o", download_file, "-d", tmp_dir], 362 cwd=install_dir) 363 # Conscrypt is not owned by the ART team, so we keep a local copy of its 364 # prebuilt with Android.bp files renamed to ArtThinBuild.bp to avoid Soong 365 # adding them as part of the build graph. 366 if "local_riscv64" in install_dir: 367 patch_cmd = ("sed -i 's|This is auto-generated. DO NOT EDIT.|" 368 "DO NOT COMMIT. Changes in this file are temporary and generated " 369 "by art/tools/buildbot-build.sh. See b/286551985.|g' {} ; ") 370 rename_cmd = 'f="{}" ; mv "$f" "$(dirname $f)"/ArtThinBuild.bp' 371 check_call(["find", "-type", "f", "-name", "Android.bp", 372 "-exec", "sh", "-c", patch_cmd + rename_cmd, ";"], 373 cwd=os.path.join(install_dir, tmp_dir)) 374 check_call(["find", "-type", "f", "-regextype", "posix-extended", 375 "-regex", ".*riscv64.*|.*Android.bp|.*ArtThinBuild.bp", 376 "-exec", "cp", "--parents", "{}", "..", ";"], 377 cwd=os.path.join(install_dir, tmp_dir)) 378 check_call(["rm", "-rf", tmp_dir], cwd=install_dir) 379 else: 380 check_call(["mkdir", install_file], cwd=install_dir) 381 # Add -DD to not extract timestamps that may confuse the build system. 382 check_call(["unzip", "-DD", download_file, "-d", install_file], 383 cwd=install_dir) 384 elif entry.unzip_single_file: 385 if not os.path.exists(unzip_file): 386 check_call(["unzip", download_file, "-d", unzip_dir, entry.unzip_single_file]) 387 check_call(["cp", unzip_file, install_file], cwd=install_dir) 388 else: 389 check_call(["cp", download_file, install_file], cwd=install_dir) 390 391 # Return a description of the source location for inclusion in the commit 392 # message. 393 return ( 394 "branch {}, target {}, build {}".format( 395 entry.source_build.branch, 396 entry.source_build.target, 397 build_number) 398 if not local_dist else None) 399 400 401def install_paths_per_git_root(roots, paths): 402 """Partitions the given paths into subpaths within the given roots. 403 404 Args: 405 roots: List of root paths. 406 paths: List of paths relative to the same directory as the root paths. 407 408 Returns: 409 A dict mapping each root to the subpaths under it. It's an error if some 410 path doesn't go into any root. 411 """ 412 res = collections.defaultdict(list) 413 for path in paths: 414 found = False 415 for root in roots: 416 if path.startswith(root + "/"): 417 res[root].append(path[len(root) + 1:]) 418 found = True 419 break 420 if not found: 421 sys.exit("Install path {} is not in any of the git roots: {}" 422 .format(path, " ".join(roots))) 423 return res 424 425 426def get_args(): 427 need_aosp_main_throttled = any( 428 source is not None and source.branch == "aosp-main-throttled" 429 for source in ([SDK_SOURCE] + list(APEX_SOURCE.values()))) 430 if need_aosp_main_throttled: 431 epilog="Either --aosp-main-build and --aosp-main-throttled-build, or --local-dist, is required." 432 else: 433 epilog="Either --aosp-main-build or --local-dist is required." 434 435 """Parses and returns command line arguments.""" 436 parser = argparse.ArgumentParser(epilog=epilog) 437 438 parser.add_argument("--aosp-main-build", metavar="NUMBER", 439 help="Build number to fetch from aosp-main") 440 parser.add_argument("--aosp-main-throttled-build", metavar="NUMBER", 441 help="Build number to fetch from aosp-main-throttled") 442 parser.add_argument("--local-dist", metavar="PATH", 443 help="Take prebuilts from this local dist dir instead of " 444 "using fetch_artifact") 445 parser.add_argument("--local-dist-riscv64", metavar="PATH", 446 help="Copy riscv64 prebuilts from a local path, which " 447 "must be $HOME/<path-to-aosp-root>/out/dist with prebuilts " 448 "already built for aosp_riscv64-trunk_staging-userdebug " 449 "target as described in README_riscv64.md. Only " 450 "riscv64-specific files and Android.bp are updated.") 451 parser.add_argument("--skip-apex", default=True, 452 action=argparse.BooleanOptionalAction, 453 help="Do not fetch .apex files.") 454 parser.add_argument("--skip-module-sdk", action="store_true", 455 help="Do not fetch and unpack sdk and module_export zips.") 456 parser.add_argument("--skip-cls", action="store_true", 457 help="Do not create branches or git commits") 458 parser.add_argument("--bug", metavar="NUMBER", 459 help="Add a 'Bug' line with this number to commit " 460 "messages.") 461 parser.add_argument("--upload", action="store_true", 462 help="Upload the CLs to Gerrit") 463 parser.add_argument("--tmp-dir", metavar="PATH", 464 help="Temporary directory for downloads. The default is " 465 "to create one and delete it when finished, but this one " 466 "will be kept, and any files already in it won't be " 467 "downloaded again.") 468 469 args = parser.parse_args() 470 471 if args.local_dist_riscv64: 472 global ARCHES 473 ARCHES = ["riscv64"] 474 args.local_dist = args.local_dist_riscv64 475 476 got_build_numbers = bool(args.aosp_main_build and 477 (args.aosp_main_throttled_build or not need_aosp_main_throttled)) 478 if ((not got_build_numbers and not args.local_dist) or 479 (got_build_numbers and args.local_dist)): 480 sys.exit(parser.format_help()) 481 482 return args 483 484 485def main(): 486 """Program entry point.""" 487 args = get_args() 488 489 if any(path for path in GIT_PROJECT_ROOTS if not os.path.exists(path)): 490 sys.exit("This script must be run in the root of the Android build tree.") 491 492 build_numbers = None 493 if args.aosp_main_build: 494 build_numbers = { 495 "aosp-main": args.aosp_main_build, 496 "aosp-main-throttled": args.aosp_main_throttled_build, 497 } 498 499 entries = install_entries() 500 501 if args.skip_apex: 502 entries = [entry for entry in entries if entry.type != "apex"] 503 if args.skip_module_sdk: 504 entries = [entry for entry in entries if entry.type != "module_sdk"] 505 if not entries: 506 sys.exit("All prebuilts skipped - nothing to do.") 507 508 install_paths = [entry.install_path for entry in entries] 509 install_paths_per_root = install_paths_per_git_root( 510 GIT_PROJECT_ROOTS, install_paths) 511 512 git_branch_name = PREBUILT_DESCR.lower().replace(" ", "-") + "-update" 513 if args.aosp_main_build: 514 git_branch_name += "-" + args.aosp_main_build 515 516 if not args.skip_cls: 517 git_paths = list(install_paths_per_root.keys()) 518 start_branch(git_branch_name, git_paths) 519 520 if not args.local_dist_riscv64: 521 for git_root, subpaths in install_paths_per_root.items(): 522 remove_files(git_root, subpaths, not args.skip_cls) 523 524 all_installed_sources = {} 525 526 tmp_dir_obj = None 527 tmp_dir = args.tmp_dir 528 if not args.tmp_dir: 529 tmp_dir_obj = tempfile.TemporaryDirectory() 530 tmp_dir = tmp_dir_obj.name 531 tmp_dir = os.path.abspath(tmp_dir) 532 try: 533 for entry in entries: 534 installed_source = install_entry( 535 tmp_dir, args.local_dist, build_numbers, entry) 536 if installed_source: 537 all_installed_sources[entry.install_path] = installed_source 538 finally: 539 if tmp_dir_obj: 540 tmp_dir_obj.cleanup() 541 542 if not args.skip_cls: 543 for git_root, subpaths in install_paths_per_root.items(): 544 installed_sources = set(src for path, src in all_installed_sources.items() 545 if path.startswith(git_root + "/")) 546 commit(git_root, PREBUILT_DESCR, sorted(list(installed_sources)), 547 subpaths, args.bug) 548 549 if args.upload: 550 # Don't upload all projects in a single repo upload call, because that 551 # makes it pop up an interactive editor. 552 for git_root in install_paths_per_root: 553 upload_branch(git_root, git_branch_name) 554 555 556if __name__ == "__main__": 557 main() 558