1#!/usr/bin/env python3 2# 3# Copyright 2018 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6 7"""Create an Android application bundle from one or more bundle modules.""" 8 9import argparse 10import concurrent.futures 11import json 12import logging 13import os 14import posixpath 15import shutil 16import sys 17from xml.etree import ElementTree 18import zipfile 19 20sys.path.append( 21 os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir))) 22from pylib.utils import dexdump 23 24import bundletool 25from util import build_utils 26from util import manifest_utils 27from util import resource_utils 28import action_helpers # build_utils adds //build to sys.path. 29import zip_helpers 30 31 32# Location of language-based assets in bundle modules. 33_LOCALES_SUBDIR = 'assets/locales/' 34 35# The fallback locale should always have its .pak file included in 36# the base apk, i.e. not use language-based asset targetting. This ensures 37# that Chrome won't crash on startup if its bundle is installed on a device 38# with an unsupported system locale (e.g. fur-rIT). 39_FALLBACK_LOCALE = 'en-US' 40 41# List of split dimensions recognized by this tool. 42_ALL_SPLIT_DIMENSIONS = [ 'ABI', 'SCREEN_DENSITY', 'LANGUAGE' ] 43 44# Due to historical reasons, certain languages identified by Chromium with a 45# 3-letters ISO 639-2 code, are mapped to a nearly equivalent 2-letters 46# ISO 639-1 code instead (due to the fact that older Android releases only 47# supported the latter when matching resources). 48# 49# the same conversion as for Java resources. 50_SHORTEN_LANGUAGE_CODE_MAP = { 51 'fil': 'tl', # Filipino to Tagalog. 52} 53 54# A list of extensions corresponding to files that should never be compressed 55# in the bundle. This used to be handled by bundletool automatically until 56# release 0.8.0, which required that this be passed to the BundleConfig 57# file instead. 58# 59# This is the original list, which was taken from aapt2, with 'webp' added to 60# it (which curiously was missing from the list). 61_UNCOMPRESSED_FILE_EXTS = [ 62 '3g2', '3gp', '3gpp', '3gpp2', 'aac', 'amr', 'awb', 'git', 'imy', 'jet', 63 'jpeg', 'jpg', 'm4a', 'm4v', 'mid', 'midi', 'mkv', 'mp2', 'mp3', 'mp4', 64 'mpeg', 'mpg', 'ogg', 'png', 'rtttl', 'smf', 'wav', 'webm', 'webp', 'wmv', 65 'xmf' 66] 67 68_COMPONENT_TYPES = ('activity', 'provider', 'receiver', 'service') 69_DEDUPE_ENTRY_TYPES = _COMPONENT_TYPES + ('activity-alias', 'meta-data') 70 71_ROTATION_METADATA_KEY = 'com.google.play.apps.signing/RotationConfig.textproto' 72 73 74def _ParseArgs(args): 75 parser = argparse.ArgumentParser() 76 parser.add_argument('--out-bundle', required=True, 77 help='Output bundle zip archive.') 78 parser.add_argument('--module-zips', required=True, 79 help='GN-list of module zip archives.') 80 parser.add_argument( 81 '--pathmap-in-paths', 82 action='append', 83 help='List of module pathmap files.') 84 parser.add_argument( 85 '--module-name', 86 action='append', 87 dest='module_names', 88 help='List of module names.') 89 parser.add_argument( 90 '--pathmap-out-path', help='Path to combined pathmap file for bundle.') 91 parser.add_argument( 92 '--rtxt-in-paths', action='append', help='GN-list of module R.txt files.') 93 parser.add_argument( 94 '--rtxt-out-path', help='Path to combined R.txt file for bundle.') 95 parser.add_argument('--uncompressed-assets', action='append', 96 help='GN-list of uncompressed assets.') 97 parser.add_argument('--compress-dex', 98 action='store_true', 99 help='Compress .dex files') 100 parser.add_argument('--split-dimensions', 101 help="GN-list of split dimensions to support.") 102 parser.add_argument( 103 '--base-module-rtxt-path', 104 help='Optional path to the base module\'s R.txt file, only used with ' 105 'language split dimension.') 106 parser.add_argument( 107 '--base-allowlist-rtxt-path', 108 help='Optional path to an R.txt file, string resources ' 109 'listed there _and_ in --base-module-rtxt-path will ' 110 'be kept in the base bundle module, even if language' 111 ' splitting is enabled.') 112 parser.add_argument('--rotation-config', 113 help='Path to a RotationConfig.textproto') 114 parser.add_argument('--warnings-as-errors', 115 action='store_true', 116 help='Treat all warnings as errors.') 117 118 parser.add_argument( 119 '--validate-services', 120 action='store_true', 121 help='Check if services are in base module if isolatedSplits is enabled.') 122 123 options = parser.parse_args(args) 124 options.module_zips = action_helpers.parse_gn_list(options.module_zips) 125 126 if len(options.module_zips) == 0: 127 parser.error('The module zip list cannot be empty.') 128 if len(options.module_zips) != len(options.module_names): 129 parser.error('# module zips != # names.') 130 if 'base' not in options.module_names: 131 parser.error('Missing base module.') 132 133 # Sort modules for more stable outputs. 134 per_module_values = list( 135 zip(options.module_names, options.module_zips, 136 options.uncompressed_assets, options.rtxt_in_paths, 137 options.pathmap_in_paths)) 138 per_module_values.sort(key=lambda x: (x[0] != 'base', x[0])) 139 options.module_names = [x[0] for x in per_module_values] 140 options.module_zips = [x[1] for x in per_module_values] 141 options.uncompressed_assets = [x[2] for x in per_module_values] 142 options.rtxt_in_paths = [x[3] for x in per_module_values] 143 options.pathmap_in_paths = [x[4] for x in per_module_values] 144 145 options.rtxt_in_paths = action_helpers.parse_gn_list(options.rtxt_in_paths) 146 options.pathmap_in_paths = action_helpers.parse_gn_list( 147 options.pathmap_in_paths) 148 149 # Merge all uncompressed assets into a set. 150 uncompressed_list = [] 151 for entry in action_helpers.parse_gn_list(options.uncompressed_assets): 152 # Each entry has the following format: 'zipPath' or 'srcPath:zipPath' 153 pos = entry.find(':') 154 if pos >= 0: 155 uncompressed_list.append(entry[pos + 1:]) 156 else: 157 uncompressed_list.append(entry) 158 159 options.uncompressed_assets = set(uncompressed_list) 160 161 # Check that all split dimensions are valid 162 if options.split_dimensions: 163 options.split_dimensions = action_helpers.parse_gn_list( 164 options.split_dimensions) 165 for dim in options.split_dimensions: 166 if dim.upper() not in _ALL_SPLIT_DIMENSIONS: 167 parser.error('Invalid split dimension "%s" (expected one of: %s)' % ( 168 dim, ', '.join(x.lower() for x in _ALL_SPLIT_DIMENSIONS))) 169 170 # As a special case, --base-allowlist-rtxt-path can be empty to indicate 171 # that the module doesn't need such a allowlist. That's because it is easier 172 # to check this condition here than through GN rules :-( 173 if options.base_allowlist_rtxt_path == '': 174 options.base_module_rtxt_path = None 175 176 # Check --base-module-rtxt-path and --base-allowlist-rtxt-path usage. 177 if options.base_module_rtxt_path: 178 if not options.base_allowlist_rtxt_path: 179 parser.error( 180 '--base-module-rtxt-path requires --base-allowlist-rtxt-path') 181 if 'language' not in options.split_dimensions: 182 parser.error('--base-module-rtxt-path is only valid with ' 183 'language-based splits.') 184 185 return options 186 187 188def _MakeSplitDimension(value, enabled): 189 """Return dict modelling a BundleConfig splitDimension entry.""" 190 return {'value': value, 'negate': not enabled} 191 192 193def _GenerateBundleConfigJson(uncompressed_assets, compress_dex, 194 split_dimensions, base_master_resource_ids): 195 """Generate a dictionary that can be written to a JSON BuildConfig. 196 197 Args: 198 uncompressed_assets: A list or set of file paths under assets/ that always 199 be stored uncompressed. 200 compressed_dex: Boolean, whether to compress .dex. 201 split_dimensions: list of split dimensions. 202 base_master_resource_ids: Optional list of 32-bit resource IDs to keep 203 inside the base module, even when split dimensions are enabled. 204 Returns: 205 A dictionary that can be written as a json file. 206 """ 207 # Compute splitsConfig list. Each item is a dictionary that can have 208 # the following keys: 209 # 'value': One of ['LANGUAGE', 'DENSITY', 'ABI'] 210 # 'negate': Boolean, True to indicate that the bundle should *not* be 211 # split (unused at the moment by this script). 212 213 split_dimensions = [ _MakeSplitDimension(dim, dim in split_dimensions) 214 for dim in _ALL_SPLIT_DIMENSIONS ] 215 216 # Locale-specific pak files stored in bundle splits need not be compressed. 217 uncompressed_globs = [ 218 'assets/locales#lang_*/*.pak', 'assets/fallback-locales/*.pak' 219 ] 220 # normpath to allow for ../ prefix. 221 uncompressed_globs.extend( 222 posixpath.normpath('assets/' + x) for x in uncompressed_assets) 223 # NOTE: Use '**' instead of '*' to work through directories! 224 uncompressed_globs.extend('**.' + ext for ext in _UNCOMPRESSED_FILE_EXTS) 225 if not compress_dex: 226 # Explicit glob required only when using bundletool to create .apks files. 227 # Play Store looks for and respects "uncompressDexFiles" set below. 228 # b/176198991 229 # This is added as a placeholder entry in order to have no effect unless 230 # processed with app_bundle_utils.GenerateBundleApks(). 231 uncompressed_globs.append('classesX.dex') 232 233 data = { 234 'optimizations': { 235 'splitsConfig': { 236 'splitDimension': split_dimensions, 237 }, 238 'uncompressNativeLibraries': { 239 'enabled': True, 240 'alignment': 'PAGE_ALIGNMENT_16K' 241 }, 242 'uncompressDexFiles': { 243 'enabled': True, # Applies only for P+. 244 } 245 }, 246 'compression': { 247 'uncompressedGlob': sorted(uncompressed_globs), 248 }, 249 } 250 251 if base_master_resource_ids: 252 data['master_resources'] = { 253 'resource_ids': list(base_master_resource_ids), 254 } 255 256 return json.dumps(data, indent=2) 257 258 259def _RewriteLanguageAssetPath(src_path): 260 """Rewrite the destination path of a locale asset for language-based splits. 261 262 Should only be used when generating bundles with language-based splits. 263 This will rewrite paths that look like locales/<locale>.pak into 264 locales#<language>/<locale>.pak, where <language> is the language code 265 from the locale. 266 267 Returns new path. 268 """ 269 if not src_path.startswith(_LOCALES_SUBDIR) or not src_path.endswith('.pak'): 270 return [src_path] 271 272 locale = src_path[len(_LOCALES_SUBDIR):-4] 273 android_locale = resource_utils.ToAndroidLocaleName(locale) 274 275 # The locale format is <lang>-<region> or <lang> or BCP-47 (e.g b+sr+Latn). 276 # Extract the language. 277 pos = android_locale.find('-') 278 if android_locale.startswith('b+'): 279 # If locale is in BCP-47 the language is the second tag (e.g. b+sr+Latn) 280 android_language = android_locale.split('+')[1] 281 elif pos >= 0: 282 android_language = android_locale[:pos] 283 else: 284 android_language = android_locale 285 286 if locale == _FALLBACK_LOCALE: 287 # Fallback locale .pak files must be placed in a different directory 288 # to ensure they are always stored in the base module. 289 result_path = 'assets/fallback-locales/%s.pak' % locale 290 else: 291 # Other language .pak files go into a language-specific asset directory 292 # that bundletool will store in separate split APKs. 293 result_path = 'assets/locales#lang_%s/%s.pak' % (android_language, locale) 294 295 return result_path 296 297 298def _SplitModuleForAssetTargeting(src_module_zip, tmp_dir, split_dimensions): 299 """Splits assets in a module if needed. 300 301 Args: 302 src_module_zip: input zip module path. 303 tmp_dir: Path to temporary directory, where the new output module might 304 be written to. 305 split_dimensions: list of split dimensions. 306 307 Returns: 308 If the module doesn't need asset targeting, doesn't do anything and 309 returns src_module_zip. Otherwise, create a new module zip archive under 310 tmp_dir with the same file name, but which contains assets paths targeting 311 the proper dimensions. 312 """ 313 split_language = 'LANGUAGE' in split_dimensions 314 if not split_language: 315 # Nothing to target, so return original module path. 316 return src_module_zip 317 318 with zipfile.ZipFile(src_module_zip, 'r') as src_zip: 319 language_files = [ 320 f for f in src_zip.namelist() if f.startswith(_LOCALES_SUBDIR)] 321 322 if not language_files: 323 # Not language-based assets to split in this module. 324 return src_module_zip 325 326 tmp_zip = os.path.join(tmp_dir, os.path.basename(src_module_zip)) 327 with zipfile.ZipFile(tmp_zip, 'w') as dst_zip: 328 for info in src_zip.infolist(): 329 src_path = info.filename 330 is_compressed = info.compress_type != zipfile.ZIP_STORED 331 332 dst_path = src_path 333 if src_path in language_files: 334 dst_path = _RewriteLanguageAssetPath(src_path) 335 336 zip_helpers.add_to_zip_hermetic(dst_zip, 337 dst_path, 338 data=src_zip.read(src_path), 339 compress=is_compressed) 340 341 return tmp_zip 342 343 344def _GenerateBaseResourcesAllowList(base_module_rtxt_path, 345 base_allowlist_rtxt_path): 346 """Generate a allowlist of base master resource ids. 347 348 Args: 349 base_module_rtxt_path: Path to base module R.txt file. 350 base_allowlist_rtxt_path: Path to base allowlist R.txt file. 351 Returns: 352 list of resource ids. 353 """ 354 ids_map = resource_utils.GenerateStringResourcesAllowList( 355 base_module_rtxt_path, base_allowlist_rtxt_path) 356 return ids_map.keys() 357 358 359def _ConcatTextFiles(in_paths, out_path): 360 """Concatenate the contents of multiple text files into one. 361 362 The each file contents is preceded by a line containing the original filename. 363 364 Args: 365 in_paths: List of input file paths. 366 out_path: Path to output file. 367 """ 368 with open(out_path, 'w') as out_file: 369 for in_path in in_paths: 370 if not os.path.exists(in_path): 371 continue 372 with open(in_path, 'r') as in_file: 373 out_file.write('-- Contents of {}\n'.format(os.path.basename(in_path))) 374 out_file.write(in_file.read()) 375 376 377def _LoadPathmap(pathmap_path): 378 """Load the pathmap of obfuscated resource paths. 379 380 Returns: A dict mapping from obfuscated paths to original paths or an 381 empty dict if passed a None |pathmap_path|. 382 """ 383 if pathmap_path is None: 384 return {} 385 386 pathmap = {} 387 with open(pathmap_path, 'r') as f: 388 for line in f: 389 line = line.strip() 390 if line.startswith('--') or line == '': 391 continue 392 original, renamed = line.split(' -> ') 393 pathmap[renamed] = original 394 return pathmap 395 396 397def _WriteBundlePathmap(module_pathmap_paths, module_names, 398 bundle_pathmap_path): 399 """Combine the contents of module pathmaps into a bundle pathmap. 400 401 This rebases the resource paths inside the module pathmap before adding them 402 to the bundle pathmap. So res/a.xml inside the base module pathmap would be 403 base/res/a.xml in the bundle pathmap. 404 """ 405 with open(bundle_pathmap_path, 'w') as bundle_pathmap_file: 406 for module_pathmap_path, module_name in zip(module_pathmap_paths, 407 module_names): 408 if not os.path.exists(module_pathmap_path): 409 continue 410 module_pathmap = _LoadPathmap(module_pathmap_path) 411 for short_path, long_path in module_pathmap.items(): 412 rebased_long_path = '{}/{}'.format(module_name, long_path) 413 rebased_short_path = '{}/{}'.format(module_name, short_path) 414 line = '{} -> {}\n'.format(rebased_long_path, rebased_short_path) 415 bundle_pathmap_file.write(line) 416 417 418def _GetManifestForModule(bundle_path, module_name): 419 data = bundletool.RunBundleTool( 420 ['dump', 'manifest', '--bundle', bundle_path, '--module', module_name]) 421 try: 422 return ElementTree.fromstring(data) 423 except ElementTree.ParseError: 424 sys.stderr.write('Failed to parse:\n') 425 sys.stderr.write(data) 426 raise 427 428 429def _GetComponentNames(manifest, tag_name): 430 android_name = '{%s}name' % manifest_utils.ANDROID_NAMESPACE 431 return [s.attrib.get(android_name) for s in manifest.iter(tag_name)] 432 433 434def _ClassesFromZip(module_zip): 435 classes = set() 436 for package in dexdump.Dump(module_zip): 437 for java_package, package_dict in package.items(): 438 java_package += '.' if java_package else '' 439 classes.update(java_package + c for c in package_dict['classes']) 440 return classes 441 442 443def _ValidateSplits(bundle_path, module_zips): 444 logging.info('Reading manifests and running dexdump') 445 base_zip = next(p for p in module_zips if os.path.basename(p) == 'base.zip') 446 module_names = sorted(os.path.basename(p)[:-len('.zip')] for p in module_zips) 447 # Using threads makes these step go from 7s -> 1s on my machine. 448 with concurrent.futures.ThreadPoolExecutor() as executor: 449 # Create list of classes from the base module's dex. 450 classes_future = executor.submit(_ClassesFromZip, base_zip) 451 452 # Create xmltrees of all module manifests. 453 manifest_futures = [ 454 executor.submit(_GetManifestForModule, bundle_path, n) 455 for n in module_names 456 ] 457 manifests_by_name = dict( 458 zip(module_names, (f.result() for f in manifest_futures))) 459 base_classes = classes_future.result() 460 461 # Collect service names from all split manifests. 462 logging.info('Performing checks') 463 errors = [] 464 465 # Ensure there are no components defined in multiple splits. 466 splits_by_component = {} 467 for module_name, cur_manifest in manifests_by_name.items(): 468 for kind in _DEDUPE_ENTRY_TYPES: 469 for component in _GetComponentNames(cur_manifest, kind): 470 owner_module_name = splits_by_component.setdefault((kind, component), 471 module_name) 472 # Allow services that exist only to keep <meta-data> out of 473 # ApplicationInfo. 474 if (owner_module_name != module_name 475 and not component.endswith('HolderService')): 476 errors.append(f'The {kind} "{component}" appeared in both ' 477 f'{owner_module_name} and {module_name}.') 478 479 # Ensure components defined in base manifest exist in base dex. 480 for (kind, component), module_name in splits_by_component.items(): 481 if module_name == 'base' and kind in _COMPONENT_TYPES: 482 if component not in base_classes: 483 errors.append(f"{component} is defined in the base manfiest, " 484 f"but the class does not exist in the base splits' dex") 485 486 # Remaining checks apply only when isolatedSplits="true". 487 isolated_splits = manifests_by_name['base'].get( 488 f'{manifest_utils.ANDROID_NAMESPACE}isolatedSplits') 489 if isolated_splits != 'true': 490 return errors 491 492 # Ensure all providers are present in base module. We enforce this because 493 # providers are loaded early in startup, and keeping them in the base module 494 # gives more time for the chrome split to load. 495 for module_name, cur_manifest in manifests_by_name.items(): 496 if module_name == 'base': 497 continue 498 provider_names = _GetComponentNames(cur_manifest, 'provider') 499 if provider_names: 500 errors.append('Providers should all be declared in the base manifest.' 501 ' "%s" module declared: %s' % (module_name, provider_names)) 502 503 # Ensure all services are present in base module because service classes are 504 # not found if they are not present in the base module. b/169196314 505 # It is fine if they are defined in split manifests though. 506 for cur_manifest in manifests_by_name.values(): 507 for service_name in _GetComponentNames(cur_manifest, 'service'): 508 if service_name not in base_classes: 509 errors.append("Service %s should be present in the base module's dex." 510 " See b/169196314 for more details." % service_name) 511 512 return errors 513 514 515def main(args): 516 build_utils.InitLogging('AAB_DEBUG') 517 args = build_utils.ExpandFileArgs(args) 518 options = _ParseArgs(args) 519 520 split_dimensions = [] 521 if options.split_dimensions: 522 split_dimensions = [x.upper() for x in options.split_dimensions] 523 524 525 with build_utils.TempDir() as tmp_dir: 526 logging.info('Splitting locale assets') 527 module_zips = [ 528 _SplitModuleForAssetTargeting(module, tmp_dir, split_dimensions) \ 529 for module in options.module_zips] 530 531 base_master_resource_ids = None 532 if options.base_module_rtxt_path: 533 logging.info('Creating R.txt allowlist') 534 base_master_resource_ids = _GenerateBaseResourcesAllowList( 535 options.base_module_rtxt_path, options.base_allowlist_rtxt_path) 536 537 logging.info('Creating BundleConfig.pb.json') 538 bundle_config = _GenerateBundleConfigJson(options.uncompressed_assets, 539 options.compress_dex, 540 split_dimensions, 541 base_master_resource_ids) 542 543 tmp_bundle = os.path.join(tmp_dir, 'tmp_bundle') 544 545 # Important: bundletool requires that the bundle config file is 546 # named with a .pb.json extension. 547 tmp_bundle_config = tmp_bundle + '.BundleConfig.pb.json' 548 549 with open(tmp_bundle_config, 'w') as f: 550 f.write(bundle_config) 551 552 logging.info('Running bundletool') 553 cmd_args = build_utils.JavaCmd() + [ 554 '-jar', 555 bundletool.BUNDLETOOL_JAR_PATH, 556 'build-bundle', 557 '--modules=' + ','.join(module_zips), 558 '--output=' + tmp_bundle, 559 '--config=' + tmp_bundle_config, 560 ] 561 562 if options.rotation_config: 563 cmd_args += [ 564 f'--metadata-file={_ROTATION_METADATA_KEY}:{options.rotation_config}' 565 ] 566 567 build_utils.CheckOutput( 568 cmd_args, 569 print_stdout=True, 570 print_stderr=True, 571 stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings, 572 fail_on_output=options.warnings_as_errors) 573 574 if options.validate_services: 575 # TODO(crbug.com/1126301): This step takes 0.4s locally for bundles with 576 # isolated splits disabled and 2s for bundles with isolated splits 577 # enabled. Consider making this run in parallel or move into a separate 578 # step before enabling isolated splits by default. 579 logging.info('Validating isolated split manifests') 580 errors = _ValidateSplits(tmp_bundle, module_zips) 581 if errors: 582 sys.stderr.write('Bundle failed sanity checks:\n ') 583 sys.stderr.write('\n '.join(errors)) 584 sys.stderr.write('\n') 585 sys.exit(1) 586 587 logging.info('Writing final output artifacts') 588 shutil.move(tmp_bundle, options.out_bundle) 589 590 if options.rtxt_out_path: 591 _ConcatTextFiles(options.rtxt_in_paths, options.rtxt_out_path) 592 593 if options.pathmap_out_path: 594 _WriteBundlePathmap(options.pathmap_in_paths, options.module_names, 595 options.pathmap_out_path) 596 597 598if __name__ == '__main__': 599 main(sys.argv[1:]) 600