1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7import argparse 8import collections 9import logging 10import os 11import re 12import shutil 13import shlex 14import sys 15import tempfile 16import zipfile 17 18from util import build_utils 19from util import md5_check 20import action_helpers # build_utils adds //build to sys.path. 21import zip_helpers 22 23 24_DEX_XMX = '2G' # Increase this when __final_dex OOMs. 25 26DEFAULT_IGNORE_WARNINGS = ( 27 # Warning: Running R8 version main (build engineering), which cannot be 28 # represented as a semantic version. Using an artificial version newer than 29 # any known version for selecting Proguard configurations embedded under 30 # META-INF/. This means that all rules with a '-upto-' qualifier will be 31 # excluded and all rules with a -from- qualifier will be included. 32 r'Running R8 version main', ) 33 34INTERFACE_DESUGARING_WARNINGS = (r'default or static interface methods', ) 35 36_SKIPPED_CLASS_FILE_NAMES = ( 37 'module-info.class', # Explicitly skipped by r8/utils/FileUtils#isClassFile 38) 39 40 41def _ParseArgs(args): 42 args = build_utils.ExpandFileArgs(args) 43 parser = argparse.ArgumentParser() 44 45 action_helpers.add_depfile_arg(parser) 46 parser.add_argument('--output', required=True, help='Dex output path.') 47 parser.add_argument( 48 '--class-inputs', 49 action='append', 50 help='GN-list of .jars with .class files.') 51 parser.add_argument( 52 '--class-inputs-filearg', 53 action='append', 54 help='GN-list of .jars with .class files (added to depfile).') 55 parser.add_argument( 56 '--dex-inputs', action='append', help='GN-list of .jars with .dex files.') 57 parser.add_argument( 58 '--dex-inputs-filearg', 59 action='append', 60 help='GN-list of .jars with .dex files (added to depfile).') 61 parser.add_argument( 62 '--incremental-dir', 63 help='Path of directory to put intermediate dex files.') 64 parser.add_argument('--library', 65 action='store_true', 66 help='Allow numerous dex files within output.') 67 parser.add_argument('--r8-jar-path', required=True, help='Path to R8 jar.') 68 parser.add_argument('--skip-custom-d8', 69 action='store_true', 70 help='When rebuilding the CustomD8 jar, this may be ' 71 'necessary to avoid incompatibility with the new r8 ' 72 'jar.') 73 parser.add_argument('--custom-d8-jar-path', 74 required=True, 75 help='Path to our customized d8 jar.') 76 parser.add_argument('--desugar-dependencies', 77 help='Path to store desugar dependencies.') 78 parser.add_argument('--desugar', action='store_true') 79 parser.add_argument( 80 '--bootclasspath', 81 action='append', 82 help='GN-list of bootclasspath. Needed for --desugar') 83 parser.add_argument('--show-desugar-default-interface-warnings', 84 action='store_true', 85 help='Enable desugaring warnings.') 86 parser.add_argument( 87 '--classpath', 88 action='append', 89 help='GN-list of full classpath. Needed for --desugar') 90 parser.add_argument('--release', 91 action='store_true', 92 help='Run D8 in release mode.') 93 parser.add_argument( 94 '--min-api', help='Minimum Android API level compatibility.') 95 parser.add_argument('--force-enable-assertions', 96 action='store_true', 97 help='Forcefully enable javac generated assertion code.') 98 parser.add_argument('--assertion-handler', 99 help='The class name of the assertion handler class.') 100 parser.add_argument('--warnings-as-errors', 101 action='store_true', 102 help='Treat all warnings as errors.') 103 parser.add_argument('--dump-inputs', 104 action='store_true', 105 help='Use when filing D8 bugs to capture inputs.' 106 ' Stores inputs to d8inputs.zip') 107 options = parser.parse_args(args) 108 109 if options.force_enable_assertions and options.assertion_handler: 110 parser.error('Cannot use both --force-enable-assertions and ' 111 '--assertion-handler') 112 113 options.class_inputs = action_helpers.parse_gn_list(options.class_inputs) 114 options.class_inputs_filearg = action_helpers.parse_gn_list( 115 options.class_inputs_filearg) 116 options.bootclasspath = action_helpers.parse_gn_list(options.bootclasspath) 117 options.classpath = action_helpers.parse_gn_list(options.classpath) 118 options.dex_inputs = action_helpers.parse_gn_list(options.dex_inputs) 119 options.dex_inputs_filearg = action_helpers.parse_gn_list( 120 options.dex_inputs_filearg) 121 122 return options 123 124 125def CreateStderrFilter(filters): 126 def filter_stderr(output): 127 # Set this when debugging R8 output. 128 if os.environ.get('R8_SHOW_ALL_OUTPUT', '0') != '0': 129 return output 130 131 # All missing definitions are logged as a single warning, but start on a 132 # new line like "Missing class ...". 133 warnings = re.split(r'^(?=Warning|Error|Missing (?:class|field|method))', 134 output, 135 flags=re.MULTILINE) 136 preamble, *warnings = warnings 137 138 combined_pattern = '|'.join(filters) 139 preamble = build_utils.FilterLines(preamble, combined_pattern) 140 141 compiled_re = re.compile(combined_pattern, re.DOTALL) 142 warnings = [w for w in warnings if not compiled_re.search(w)] 143 144 return preamble + ''.join(warnings) 145 146 return filter_stderr 147 148 149def _RunD8(dex_cmd, input_paths, output_path, warnings_as_errors, 150 show_desugar_default_interface_warnings): 151 dex_cmd = dex_cmd + ['--output', output_path] + input_paths 152 153 # Missing deps can happen for prebuilts that are missing transitive deps 154 # and have set enable_bytecode_checks=false. 155 filters = list(DEFAULT_IGNORE_WARNINGS) 156 if not show_desugar_default_interface_warnings: 157 filters += INTERFACE_DESUGARING_WARNINGS 158 159 stderr_filter = CreateStderrFilter(filters) 160 161 is_debug = logging.getLogger().isEnabledFor(logging.DEBUG) 162 163 # Avoid deleting the flag file when DEX_DEBUG is set in case the flag file 164 # needs to be examined after the build. 165 with tempfile.NamedTemporaryFile(mode='w', delete=not is_debug) as flag_file: 166 # Chosen arbitrarily. Needed to avoid command-line length limits. 167 MAX_ARGS = 50 168 orig_dex_cmd = dex_cmd 169 if len(dex_cmd) > MAX_ARGS: 170 # Add all flags to D8 (anything after the first --) as well as all 171 # positional args at the end to the flag file. 172 for idx, cmd in enumerate(dex_cmd): 173 if cmd.startswith('--'): 174 flag_file.write('\n'.join(dex_cmd[idx:])) 175 flag_file.flush() 176 dex_cmd = dex_cmd[:idx] 177 dex_cmd.append('@' + flag_file.name) 178 break 179 180 # stdout sometimes spams with things like: 181 # Stripped invalid locals information from 1 method. 182 try: 183 build_utils.CheckOutput(dex_cmd, 184 stderr_filter=stderr_filter, 185 fail_on_output=warnings_as_errors) 186 except Exception as e: 187 if isinstance(e, build_utils.CalledProcessError): 188 output = e.output # pylint: disable=no-member 189 if "global synthetic for 'Record desugaring'" in output: 190 sys.stderr.write('Java records are not supported.\n') 191 sys.stderr.write( 192 'See https://chromium.googlesource.com/chromium/src/+/' + 193 'main/styleguide/java/java.md#Records\n') 194 sys.exit(1) 195 if orig_dex_cmd is not dex_cmd: 196 sys.stderr.write('Full command: ' + shlex.join(orig_dex_cmd) + '\n') 197 raise 198 199 200def _ZipAligned(dex_files, output_path): 201 """Creates a .dex.jar with 4-byte aligned files. 202 203 Args: 204 dex_files: List of dex files. 205 output_path: The output file in which to write the zip. 206 """ 207 with zipfile.ZipFile(output_path, 'w') as z: 208 for i, dex_file in enumerate(dex_files): 209 name = 'classes{}.dex'.format(i + 1 if i > 0 else '') 210 zip_helpers.add_to_zip_hermetic(z, name, src_path=dex_file, alignment=4) 211 212 213def _CreateFinalDex(d8_inputs, output, tmp_dir, dex_cmd, options=None): 214 tmp_dex_output = os.path.join(tmp_dir, 'tmp_dex_output.zip') 215 needs_dexing = not all(f.endswith('.dex') for f in d8_inputs) 216 needs_dexmerge = output.endswith('.dex') or not (options and options.library) 217 if needs_dexing or needs_dexmerge: 218 tmp_dex_dir = os.path.join(tmp_dir, 'tmp_dex_dir') 219 os.mkdir(tmp_dex_dir) 220 221 _RunD8(dex_cmd, d8_inputs, tmp_dex_dir, 222 (not options or options.warnings_as_errors), 223 (options and options.show_desugar_default_interface_warnings)) 224 logging.debug('Performed dex merging') 225 226 dex_files = [os.path.join(tmp_dex_dir, f) for f in os.listdir(tmp_dex_dir)] 227 228 if output.endswith('.dex'): 229 if len(dex_files) > 1: 230 raise Exception('%d files created, expected 1' % len(dex_files)) 231 tmp_dex_output = dex_files[0] 232 else: 233 _ZipAligned(sorted(dex_files), tmp_dex_output) 234 else: 235 # Skip dexmerger. Just put all incrementals into the .jar individually. 236 _ZipAligned(sorted(d8_inputs), tmp_dex_output) 237 logging.debug('Quick-zipped %d files', len(d8_inputs)) 238 239 # The dex file is complete and can be moved out of tmp_dir. 240 shutil.move(tmp_dex_output, output) 241 242 243def _IntermediateDexFilePathsFromInputJars(class_inputs, incremental_dir): 244 """Returns a list of all intermediate dex file paths.""" 245 dex_files = [] 246 for jar in class_inputs: 247 with zipfile.ZipFile(jar, 'r') as z: 248 for subpath in z.namelist(): 249 if _IsClassFile(subpath): 250 subpath = subpath[:-5] + 'dex' 251 dex_files.append(os.path.join(incremental_dir, subpath)) 252 return dex_files 253 254 255def _DeleteStaleIncrementalDexFiles(dex_dir, dex_files): 256 """Deletes intermediate .dex files that are no longer needed.""" 257 all_files = build_utils.FindInDirectory(dex_dir) 258 desired_files = set(dex_files) 259 for path in all_files: 260 if path not in desired_files: 261 os.unlink(path) 262 263 264def _ParseDesugarDeps(desugar_dependencies_file): 265 # pylint: disable=line-too-long 266 """Returns a dict of dependent/dependency mapping parsed from the file. 267 268 Example file format: 269 $ tail out/Debug/gen/base/base_java__dex.desugardeps 270 org/chromium/base/task/SingleThreadTaskRunnerImpl.class 271 <- org/chromium/base/task/SingleThreadTaskRunner.class 272 <- org/chromium/base/task/TaskRunnerImpl.class 273 org/chromium/base/task/TaskRunnerImpl.class 274 <- org/chromium/base/task/TaskRunner.class 275 org/chromium/base/task/TaskRunnerImplJni$1.class 276 <- obj/base/jni_java.turbine.jar:org/jni_zero/JniStaticTestMocker.class 277 org/chromium/base/task/TaskRunnerImplJni.class 278 <- org/chromium/base/task/TaskRunnerImpl$Natives.class 279 """ 280 # pylint: enable=line-too-long 281 dependents_from_dependency = collections.defaultdict(set) 282 if desugar_dependencies_file and os.path.exists(desugar_dependencies_file): 283 with open(desugar_dependencies_file, 'r') as f: 284 dependent = None 285 for line in f: 286 line = line.rstrip() 287 if line.startswith(' <- '): 288 dependency = line[len(' <- '):] 289 # Note that this is a reversed mapping from the one in CustomD8.java. 290 dependents_from_dependency[dependency].add(dependent) 291 else: 292 dependent = line 293 return dependents_from_dependency 294 295 296def _ComputeRequiredDesugarClasses(changes, desugar_dependencies_file, 297 class_inputs, classpath): 298 dependents_from_dependency = _ParseDesugarDeps(desugar_dependencies_file) 299 required_classes = set() 300 # Gather classes that need to be re-desugared from changes in the classpath. 301 for jar in classpath: 302 for subpath in changes.IterChangedSubpaths(jar): 303 dependency = '{}:{}'.format(jar, subpath) 304 required_classes.update(dependents_from_dependency[dependency]) 305 306 for jar in class_inputs: 307 for subpath in changes.IterChangedSubpaths(jar): 308 required_classes.update(dependents_from_dependency[subpath]) 309 310 return required_classes 311 312 313def _IsClassFile(path): 314 if os.path.basename(path) in _SKIPPED_CLASS_FILE_NAMES: 315 return False 316 return path.endswith('.class') 317 318 319def _ExtractClassFiles(changes, tmp_dir, class_inputs, required_classes_set): 320 classes_list = [] 321 for jar in class_inputs: 322 if changes: 323 changed_class_list = (set(changes.IterChangedSubpaths(jar)) 324 | required_classes_set) 325 predicate = lambda x: x in changed_class_list and _IsClassFile(x) 326 else: 327 predicate = _IsClassFile 328 329 classes_list.extend( 330 build_utils.ExtractAll(jar, path=tmp_dir, predicate=predicate)) 331 return classes_list 332 333 334def _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd): 335 # Create temporary directory for classes to be extracted to. 336 tmp_extract_dir = os.path.join(tmp_dir, 'tmp_extract_dir') 337 os.mkdir(tmp_extract_dir) 338 339 # Do a full rebuild when changes occur in non-input files. 340 allowed_changed = set(options.class_inputs) 341 allowed_changed.update(options.dex_inputs) 342 allowed_changed.update(options.classpath) 343 strings_changed = changes.HasStringChanges() 344 non_direct_input_changed = next( 345 (p for p in changes.IterChangedPaths() if p not in allowed_changed), None) 346 347 if strings_changed or non_direct_input_changed: 348 logging.debug('Full dex required: strings_changed=%s path_changed=%s', 349 strings_changed, non_direct_input_changed) 350 changes = None 351 352 if changes is None: 353 required_desugar_classes_set = set() 354 else: 355 required_desugar_classes_set = _ComputeRequiredDesugarClasses( 356 changes, options.desugar_dependencies, options.class_inputs, 357 options.classpath) 358 logging.debug('Class files needing re-desugar: %d', 359 len(required_desugar_classes_set)) 360 class_files = _ExtractClassFiles(changes, tmp_extract_dir, 361 options.class_inputs, 362 required_desugar_classes_set) 363 logging.debug('Extracted class files: %d', len(class_files)) 364 365 # If the only change is deleting a file, class_files will be empty. 366 if class_files: 367 # Dex necessary classes into intermediate dex files. 368 dex_cmd = dex_cmd + ['--intermediate', '--file-per-class-file'] 369 if options.desugar_dependencies and not options.skip_custom_d8: 370 # Adding os.sep to remove the entire prefix. 371 dex_cmd += ['--file-tmp-prefix', tmp_extract_dir + os.sep] 372 if changes is None and os.path.exists(options.desugar_dependencies): 373 # Since incremental dexing only ever adds to the desugar_dependencies 374 # file, whenever full dexes are required the .desugardeps files need to 375 # be manually removed. 376 os.unlink(options.desugar_dependencies) 377 _RunD8(dex_cmd, class_files, options.incremental_dir, 378 options.warnings_as_errors, 379 options.show_desugar_default_interface_warnings) 380 logging.debug('Dexed class files.') 381 382 383def _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd): 384 logging.debug('_OnStaleMd5') 385 with build_utils.TempDir() as tmp_dir: 386 if options.incremental_dir: 387 # Create directory for all intermediate dex files. 388 if not os.path.exists(options.incremental_dir): 389 os.makedirs(options.incremental_dir) 390 391 _DeleteStaleIncrementalDexFiles(options.incremental_dir, final_dex_inputs) 392 logging.debug('Stale files deleted') 393 _CreateIntermediateDexFiles(changes, options, tmp_dir, dex_cmd) 394 395 _CreateFinalDex( 396 final_dex_inputs, options.output, tmp_dir, dex_cmd, options=options) 397 398 399def MergeDexForIncrementalInstall(r8_jar_path, src_paths, dest_dex_jar, 400 min_api): 401 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) + [ 402 '-cp', 403 r8_jar_path, 404 'com.android.tools.r8.D8', 405 '--min-api', 406 min_api, 407 ] 408 with build_utils.TempDir() as tmp_dir: 409 _CreateFinalDex(src_paths, dest_dex_jar, tmp_dir, dex_cmd) 410 411 412def main(args): 413 build_utils.InitLogging('DEX_DEBUG') 414 options = _ParseArgs(args) 415 416 options.class_inputs += options.class_inputs_filearg 417 options.dex_inputs += options.dex_inputs_filearg 418 419 input_paths = ([ 420 build_utils.JAVA_PATH_FOR_INPUTS, options.r8_jar_path, 421 options.custom_d8_jar_path 422 ] + options.class_inputs + options.dex_inputs) 423 424 depfile_deps = options.class_inputs_filearg + options.dex_inputs_filearg 425 426 output_paths = [options.output] 427 428 track_subpaths_allowlist = [] 429 if options.incremental_dir: 430 final_dex_inputs = _IntermediateDexFilePathsFromInputJars( 431 options.class_inputs, options.incremental_dir) 432 output_paths += final_dex_inputs 433 track_subpaths_allowlist += options.class_inputs 434 else: 435 final_dex_inputs = list(options.class_inputs) 436 final_dex_inputs += options.dex_inputs 437 438 dex_cmd = build_utils.JavaCmd(xmx=_DEX_XMX) 439 440 if options.dump_inputs: 441 dex_cmd += ['-Dcom.android.tools.r8.dumpinputtofile=d8inputs.zip'] 442 443 if not options.skip_custom_d8: 444 dex_cmd += [ 445 '-cp', 446 '{}:{}'.format(options.r8_jar_path, options.custom_d8_jar_path), 447 'org.chromium.build.CustomD8', 448 ] 449 else: 450 dex_cmd += [ 451 '-cp', 452 options.r8_jar_path, 453 'com.android.tools.r8.D8', 454 ] 455 456 if options.release: 457 dex_cmd += ['--release'] 458 if options.min_api: 459 dex_cmd += ['--min-api', options.min_api] 460 461 if not options.desugar: 462 dex_cmd += ['--no-desugaring'] 463 elif options.classpath: 464 # The classpath is used by D8 to for interface desugaring. 465 if options.desugar_dependencies and not options.skip_custom_d8: 466 dex_cmd += ['--desugar-dependencies', options.desugar_dependencies] 467 if track_subpaths_allowlist: 468 track_subpaths_allowlist += options.classpath 469 depfile_deps += options.classpath 470 input_paths += options.classpath 471 # Still pass the entire classpath in case a new dependency is needed by 472 # desugar, so that desugar_dependencies will be updated for the next build. 473 for path in options.classpath: 474 dex_cmd += ['--classpath', path] 475 476 if options.classpath: 477 dex_cmd += ['--lib', build_utils.JAVA_HOME] 478 for path in options.bootclasspath: 479 dex_cmd += ['--lib', path] 480 depfile_deps += options.bootclasspath 481 input_paths += options.bootclasspath 482 483 484 if options.assertion_handler: 485 dex_cmd += ['--force-assertions-handler:' + options.assertion_handler] 486 if options.force_enable_assertions: 487 dex_cmd += ['--force-enable-assertions'] 488 489 # The changes feature from md5_check allows us to only re-dex the class files 490 # that have changed and the class files that need to be re-desugared by D8. 491 md5_check.CallAndWriteDepfileIfStale( 492 lambda changes: _OnStaleMd5(changes, options, final_dex_inputs, dex_cmd), 493 options, 494 input_paths=input_paths, 495 input_strings=dex_cmd + [str(bool(options.incremental_dir))], 496 output_paths=output_paths, 497 pass_changes=True, 498 track_subpaths_allowlist=track_subpaths_allowlist, 499 depfile_deps=depfile_deps) 500 501 502if __name__ == '__main__': 503 sys.exit(main(sys.argv[1:])) 504