xref: /aosp_15_r20/external/cronet/build/android/gyp/create_app_bundle.py (revision 6777b5387eb2ff775bb5750e3f5d96f37fb7352b)
1#!/usr/bin/env python3
2#
3# Copyright 2018 The Chromium Authors
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Create an Android application bundle from one or more bundle modules."""
8
9import argparse
10import concurrent.futures
11import json
12import logging
13import os
14import posixpath
15import shutil
16import sys
17from xml.etree import ElementTree
18import zipfile
19
20sys.path.append(
21    os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir)))
22from pylib.utils import dexdump
23
24import bundletool
25from util import build_utils
26from util import manifest_utils
27from util import resource_utils
28import action_helpers  # build_utils adds //build to sys.path.
29import zip_helpers
30
31
32# Location of language-based assets in bundle modules.
33_LOCALES_SUBDIR = 'assets/locales/'
34
35# The fallback locale should always have its .pak file included in
36# the base apk, i.e. not use language-based asset targetting. This ensures
37# that Chrome won't crash on startup if its bundle is installed on a device
38# with an unsupported system locale (e.g. fur-rIT).
39_FALLBACK_LOCALE = 'en-US'
40
41# List of split dimensions recognized by this tool.
42_ALL_SPLIT_DIMENSIONS = [ 'ABI', 'SCREEN_DENSITY', 'LANGUAGE' ]
43
44# Due to historical reasons, certain languages identified by Chromium with a
45# 3-letters ISO 639-2 code, are mapped to a nearly equivalent 2-letters
46# ISO 639-1 code instead (due to the fact that older Android releases only
47# supported the latter when matching resources).
48#
49# the same conversion as for Java resources.
50_SHORTEN_LANGUAGE_CODE_MAP = {
51  'fil': 'tl',  # Filipino to Tagalog.
52}
53
54# A list of extensions corresponding to files that should never be compressed
55# in the bundle. This used to be handled by bundletool automatically until
56# release 0.8.0, which required that this be passed to the BundleConfig
57# file instead.
58#
59# This is the original list, which was taken from aapt2, with 'webp' added to
60# it (which curiously was missing from the list).
61_UNCOMPRESSED_FILE_EXTS = [
62    '3g2', '3gp', '3gpp', '3gpp2', 'aac', 'amr', 'awb', 'git', 'imy', 'jet',
63    'jpeg', 'jpg', 'm4a', 'm4v', 'mid', 'midi', 'mkv', 'mp2', 'mp3', 'mp4',
64    'mpeg', 'mpg', 'ogg', 'png', 'rtttl', 'smf', 'wav', 'webm', 'webp', 'wmv',
65    'xmf'
66]
67
68_COMPONENT_TYPES = ('activity', 'provider', 'receiver', 'service')
69_DEDUPE_ENTRY_TYPES = _COMPONENT_TYPES + ('activity-alias', 'meta-data')
70
71_ROTATION_METADATA_KEY = 'com.google.play.apps.signing/RotationConfig.textproto'
72
73
74def _ParseArgs(args):
75  parser = argparse.ArgumentParser()
76  parser.add_argument('--out-bundle', required=True,
77                      help='Output bundle zip archive.')
78  parser.add_argument('--module-zips', required=True,
79                      help='GN-list of module zip archives.')
80  parser.add_argument(
81      '--pathmap-in-paths',
82      action='append',
83      help='List of module pathmap files.')
84  parser.add_argument(
85      '--module-name',
86      action='append',
87      dest='module_names',
88      help='List of module names.')
89  parser.add_argument(
90      '--pathmap-out-path', help='Path to combined pathmap file for bundle.')
91  parser.add_argument(
92      '--rtxt-in-paths', action='append', help='GN-list of module R.txt files.')
93  parser.add_argument(
94      '--rtxt-out-path', help='Path to combined R.txt file for bundle.')
95  parser.add_argument('--uncompressed-assets', action='append',
96                      help='GN-list of uncompressed assets.')
97  parser.add_argument('--compress-dex',
98                      action='store_true',
99                      help='Compress .dex files')
100  parser.add_argument('--split-dimensions',
101                      help="GN-list of split dimensions to support.")
102  parser.add_argument(
103      '--base-module-rtxt-path',
104      help='Optional path to the base module\'s R.txt file, only used with '
105      'language split dimension.')
106  parser.add_argument(
107      '--base-allowlist-rtxt-path',
108      help='Optional path to an R.txt file, string resources '
109      'listed there _and_ in --base-module-rtxt-path will '
110      'be kept in the base bundle module, even if language'
111      ' splitting is enabled.')
112  parser.add_argument('--rotation-config',
113                      help='Path to a RotationConfig.textproto')
114  parser.add_argument('--warnings-as-errors',
115                      action='store_true',
116                      help='Treat all warnings as errors.')
117
118  parser.add_argument(
119      '--validate-services',
120      action='store_true',
121      help='Check if services are in base module if isolatedSplits is enabled.')
122
123  options = parser.parse_args(args)
124  options.module_zips = action_helpers.parse_gn_list(options.module_zips)
125
126  if len(options.module_zips) == 0:
127    parser.error('The module zip list cannot be empty.')
128  if len(options.module_zips) != len(options.module_names):
129    parser.error('# module zips != # names.')
130  if 'base' not in options.module_names:
131    parser.error('Missing base module.')
132
133  # Sort modules for more stable outputs.
134  per_module_values = list(
135      zip(options.module_names, options.module_zips,
136          options.uncompressed_assets, options.rtxt_in_paths,
137          options.pathmap_in_paths))
138  per_module_values.sort(key=lambda x: (x[0] != 'base', x[0]))
139  options.module_names = [x[0] for x in per_module_values]
140  options.module_zips = [x[1] for x in per_module_values]
141  options.uncompressed_assets = [x[2] for x in per_module_values]
142  options.rtxt_in_paths = [x[3] for x in per_module_values]
143  options.pathmap_in_paths = [x[4] for x in per_module_values]
144
145  options.rtxt_in_paths = action_helpers.parse_gn_list(options.rtxt_in_paths)
146  options.pathmap_in_paths = action_helpers.parse_gn_list(
147      options.pathmap_in_paths)
148
149  # Merge all uncompressed assets into a set.
150  uncompressed_list = []
151  for entry in action_helpers.parse_gn_list(options.uncompressed_assets):
152    # Each entry has the following format: 'zipPath' or 'srcPath:zipPath'
153    pos = entry.find(':')
154    if pos >= 0:
155      uncompressed_list.append(entry[pos + 1:])
156    else:
157      uncompressed_list.append(entry)
158
159  options.uncompressed_assets = set(uncompressed_list)
160
161  # Check that all split dimensions are valid
162  if options.split_dimensions:
163    options.split_dimensions = action_helpers.parse_gn_list(
164        options.split_dimensions)
165    for dim in options.split_dimensions:
166      if dim.upper() not in _ALL_SPLIT_DIMENSIONS:
167        parser.error('Invalid split dimension "%s" (expected one of: %s)' % (
168            dim, ', '.join(x.lower() for x in _ALL_SPLIT_DIMENSIONS)))
169
170  # As a special case, --base-allowlist-rtxt-path can be empty to indicate
171  # that the module doesn't need such a allowlist. That's because it is easier
172  # to check this condition here than through GN rules :-(
173  if options.base_allowlist_rtxt_path == '':
174    options.base_module_rtxt_path = None
175
176  # Check --base-module-rtxt-path and --base-allowlist-rtxt-path usage.
177  if options.base_module_rtxt_path:
178    if not options.base_allowlist_rtxt_path:
179      parser.error(
180          '--base-module-rtxt-path requires --base-allowlist-rtxt-path')
181    if 'language' not in options.split_dimensions:
182      parser.error('--base-module-rtxt-path is only valid with '
183                   'language-based splits.')
184
185  return options
186
187
188def _MakeSplitDimension(value, enabled):
189  """Return dict modelling a BundleConfig splitDimension entry."""
190  return {'value': value, 'negate': not enabled}
191
192
193def _GenerateBundleConfigJson(uncompressed_assets, compress_dex,
194                              split_dimensions, base_master_resource_ids):
195  """Generate a dictionary that can be written to a JSON BuildConfig.
196
197  Args:
198    uncompressed_assets: A list or set of file paths under assets/ that always
199      be stored uncompressed.
200    compressed_dex: Boolean, whether to compress .dex.
201    split_dimensions: list of split dimensions.
202    base_master_resource_ids: Optional list of 32-bit resource IDs to keep
203      inside the base module, even when split dimensions are enabled.
204  Returns:
205    A dictionary that can be written as a json file.
206  """
207  # Compute splitsConfig list. Each item is a dictionary that can have
208  # the following keys:
209  #    'value': One of ['LANGUAGE', 'DENSITY', 'ABI']
210  #    'negate': Boolean, True to indicate that the bundle should *not* be
211  #              split (unused at the moment by this script).
212
213  split_dimensions = [ _MakeSplitDimension(dim, dim in split_dimensions)
214                       for dim in _ALL_SPLIT_DIMENSIONS ]
215
216  # Locale-specific pak files stored in bundle splits need not be compressed.
217  uncompressed_globs = [
218      'assets/locales#lang_*/*.pak', 'assets/fallback-locales/*.pak'
219  ]
220  # normpath to allow for ../ prefix.
221  uncompressed_globs.extend(
222      posixpath.normpath('assets/' + x) for x in uncompressed_assets)
223  # NOTE: Use '**' instead of '*' to work through directories!
224  uncompressed_globs.extend('**.' + ext for ext in _UNCOMPRESSED_FILE_EXTS)
225  if not compress_dex:
226    # Explicit glob required only when using bundletool to create .apks files.
227    # Play Store looks for and respects "uncompressDexFiles" set below.
228    # b/176198991
229    # This is added as a placeholder entry in order to have no effect unless
230    # processed with app_bundle_utils.GenerateBundleApks().
231    uncompressed_globs.append('classesX.dex')
232
233  data = {
234      'optimizations': {
235          'splitsConfig': {
236              'splitDimension': split_dimensions,
237          },
238          'uncompressNativeLibraries': {
239              'enabled': True,
240              'alignment': 'PAGE_ALIGNMENT_16K'
241          },
242          'uncompressDexFiles': {
243              'enabled': True,  # Applies only for P+.
244          }
245      },
246      'compression': {
247          'uncompressedGlob': sorted(uncompressed_globs),
248      },
249  }
250
251  if base_master_resource_ids:
252    data['master_resources'] = {
253        'resource_ids': list(base_master_resource_ids),
254    }
255
256  return json.dumps(data, indent=2)
257
258
259def _RewriteLanguageAssetPath(src_path):
260  """Rewrite the destination path of a locale asset for language-based splits.
261
262  Should only be used when generating bundles with language-based splits.
263  This will rewrite paths that look like locales/<locale>.pak into
264  locales#<language>/<locale>.pak, where <language> is the language code
265  from the locale.
266
267  Returns new path.
268  """
269  if not src_path.startswith(_LOCALES_SUBDIR) or not src_path.endswith('.pak'):
270    return [src_path]
271
272  locale = src_path[len(_LOCALES_SUBDIR):-4]
273  android_locale = resource_utils.ToAndroidLocaleName(locale)
274
275  # The locale format is <lang>-<region> or <lang> or BCP-47 (e.g b+sr+Latn).
276  # Extract the language.
277  pos = android_locale.find('-')
278  if android_locale.startswith('b+'):
279    # If locale is in BCP-47 the language is the second tag (e.g. b+sr+Latn)
280    android_language = android_locale.split('+')[1]
281  elif pos >= 0:
282    android_language = android_locale[:pos]
283  else:
284    android_language = android_locale
285
286  if locale == _FALLBACK_LOCALE:
287    # Fallback locale .pak files must be placed in a different directory
288    # to ensure they are always stored in the base module.
289    result_path = 'assets/fallback-locales/%s.pak' % locale
290  else:
291    # Other language .pak files go into a language-specific asset directory
292    # that bundletool will store in separate split APKs.
293    result_path = 'assets/locales#lang_%s/%s.pak' % (android_language, locale)
294
295  return result_path
296
297
298def _SplitModuleForAssetTargeting(src_module_zip, tmp_dir, split_dimensions):
299  """Splits assets in a module if needed.
300
301  Args:
302    src_module_zip: input zip module path.
303    tmp_dir: Path to temporary directory, where the new output module might
304      be written to.
305    split_dimensions: list of split dimensions.
306
307  Returns:
308    If the module doesn't need asset targeting, doesn't do anything and
309    returns src_module_zip. Otherwise, create a new module zip archive under
310    tmp_dir with the same file name, but which contains assets paths targeting
311    the proper dimensions.
312  """
313  split_language = 'LANGUAGE' in split_dimensions
314  if not split_language:
315    # Nothing to target, so return original module path.
316    return src_module_zip
317
318  with zipfile.ZipFile(src_module_zip, 'r') as src_zip:
319    language_files = [
320      f for f in src_zip.namelist() if f.startswith(_LOCALES_SUBDIR)]
321
322    if not language_files:
323      # Not language-based assets to split in this module.
324      return src_module_zip
325
326    tmp_zip = os.path.join(tmp_dir, os.path.basename(src_module_zip))
327    with zipfile.ZipFile(tmp_zip, 'w') as dst_zip:
328      for info in src_zip.infolist():
329        src_path = info.filename
330        is_compressed = info.compress_type != zipfile.ZIP_STORED
331
332        dst_path = src_path
333        if src_path in language_files:
334          dst_path = _RewriteLanguageAssetPath(src_path)
335
336        zip_helpers.add_to_zip_hermetic(dst_zip,
337                                        dst_path,
338                                        data=src_zip.read(src_path),
339                                        compress=is_compressed)
340
341    return tmp_zip
342
343
344def _GenerateBaseResourcesAllowList(base_module_rtxt_path,
345                                    base_allowlist_rtxt_path):
346  """Generate a allowlist of base master resource ids.
347
348  Args:
349    base_module_rtxt_path: Path to base module R.txt file.
350    base_allowlist_rtxt_path: Path to base allowlist R.txt file.
351  Returns:
352    list of resource ids.
353  """
354  ids_map = resource_utils.GenerateStringResourcesAllowList(
355      base_module_rtxt_path, base_allowlist_rtxt_path)
356  return ids_map.keys()
357
358
359def _ConcatTextFiles(in_paths, out_path):
360  """Concatenate the contents of multiple text files into one.
361
362  The each file contents is preceded by a line containing the original filename.
363
364  Args:
365    in_paths: List of input file paths.
366    out_path: Path to output file.
367  """
368  with open(out_path, 'w') as out_file:
369    for in_path in in_paths:
370      if not os.path.exists(in_path):
371        continue
372      with open(in_path, 'r') as in_file:
373        out_file.write('-- Contents of {}\n'.format(os.path.basename(in_path)))
374        out_file.write(in_file.read())
375
376
377def _LoadPathmap(pathmap_path):
378  """Load the pathmap of obfuscated resource paths.
379
380  Returns: A dict mapping from obfuscated paths to original paths or an
381           empty dict if passed a None |pathmap_path|.
382  """
383  if pathmap_path is None:
384    return {}
385
386  pathmap = {}
387  with open(pathmap_path, 'r') as f:
388    for line in f:
389      line = line.strip()
390      if line.startswith('--') or line == '':
391        continue
392      original, renamed = line.split(' -> ')
393      pathmap[renamed] = original
394  return pathmap
395
396
397def _WriteBundlePathmap(module_pathmap_paths, module_names,
398                        bundle_pathmap_path):
399  """Combine the contents of module pathmaps into a bundle pathmap.
400
401  This rebases the resource paths inside the module pathmap before adding them
402  to the bundle pathmap. So res/a.xml inside the base module pathmap would be
403  base/res/a.xml in the bundle pathmap.
404  """
405  with open(bundle_pathmap_path, 'w') as bundle_pathmap_file:
406    for module_pathmap_path, module_name in zip(module_pathmap_paths,
407                                                module_names):
408      if not os.path.exists(module_pathmap_path):
409        continue
410      module_pathmap = _LoadPathmap(module_pathmap_path)
411      for short_path, long_path in module_pathmap.items():
412        rebased_long_path = '{}/{}'.format(module_name, long_path)
413        rebased_short_path = '{}/{}'.format(module_name, short_path)
414        line = '{} -> {}\n'.format(rebased_long_path, rebased_short_path)
415        bundle_pathmap_file.write(line)
416
417
418def _GetManifestForModule(bundle_path, module_name):
419  data = bundletool.RunBundleTool(
420      ['dump', 'manifest', '--bundle', bundle_path, '--module', module_name])
421  try:
422    return ElementTree.fromstring(data)
423  except ElementTree.ParseError:
424    sys.stderr.write('Failed to parse:\n')
425    sys.stderr.write(data)
426    raise
427
428
429def _GetComponentNames(manifest, tag_name):
430  android_name = '{%s}name' % manifest_utils.ANDROID_NAMESPACE
431  return [s.attrib.get(android_name) for s in manifest.iter(tag_name)]
432
433
434def _ClassesFromZip(module_zip):
435  classes = set()
436  for package in dexdump.Dump(module_zip):
437    for java_package, package_dict in package.items():
438      java_package += '.' if java_package else ''
439      classes.update(java_package + c for c in package_dict['classes'])
440  return classes
441
442
443def _ValidateSplits(bundle_path, module_zips):
444  logging.info('Reading manifests and running dexdump')
445  base_zip = next(p for p in module_zips if os.path.basename(p) == 'base.zip')
446  module_names = sorted(os.path.basename(p)[:-len('.zip')] for p in module_zips)
447  # Using threads makes these step go from 7s -> 1s on my machine.
448  with concurrent.futures.ThreadPoolExecutor() as executor:
449    # Create list of classes from the base module's dex.
450    classes_future = executor.submit(_ClassesFromZip, base_zip)
451
452    # Create xmltrees of all module manifests.
453    manifest_futures = [
454        executor.submit(_GetManifestForModule, bundle_path, n)
455        for n in module_names
456    ]
457    manifests_by_name = dict(
458        zip(module_names, (f.result() for f in manifest_futures)))
459    base_classes = classes_future.result()
460
461  # Collect service names from all split manifests.
462  logging.info('Performing checks')
463  errors = []
464
465  # Ensure there are no components defined in multiple splits.
466  splits_by_component = {}
467  for module_name, cur_manifest in manifests_by_name.items():
468    for kind in _DEDUPE_ENTRY_TYPES:
469      for component in _GetComponentNames(cur_manifest, kind):
470        owner_module_name = splits_by_component.setdefault((kind, component),
471                                                           module_name)
472        # Allow services that exist only to keep <meta-data> out of
473        # ApplicationInfo.
474        if (owner_module_name != module_name
475            and not component.endswith('HolderService')):
476          errors.append(f'The {kind} "{component}" appeared in both '
477                        f'{owner_module_name} and {module_name}.')
478
479  # Ensure components defined in base manifest exist in base dex.
480  for (kind, component), module_name in splits_by_component.items():
481    if module_name == 'base' and kind in _COMPONENT_TYPES:
482      if component not in base_classes:
483        errors.append(f"{component} is defined in the base manfiest, "
484                      f"but the class does not exist in the base splits' dex")
485
486  # Remaining checks apply only when isolatedSplits="true".
487  isolated_splits = manifests_by_name['base'].get(
488      f'{manifest_utils.ANDROID_NAMESPACE}isolatedSplits')
489  if isolated_splits != 'true':
490    return errors
491
492  # Ensure all providers are present in base module. We enforce this because
493  # providers are loaded early in startup, and keeping them in the base module
494  # gives more time for the chrome split to load.
495  for module_name, cur_manifest in manifests_by_name.items():
496    if module_name == 'base':
497      continue
498    provider_names = _GetComponentNames(cur_manifest, 'provider')
499    if provider_names:
500      errors.append('Providers should all be declared in the base manifest.'
501                    ' "%s" module declared: %s' % (module_name, provider_names))
502
503  # Ensure all services are present in base module because service classes are
504  # not found if they are not present in the base module. b/169196314
505  # It is fine if they are defined in split manifests though.
506  for cur_manifest in manifests_by_name.values():
507    for service_name in _GetComponentNames(cur_manifest, 'service'):
508      if service_name not in base_classes:
509        errors.append("Service %s should be present in the base module's dex."
510                      " See b/169196314 for more details." % service_name)
511
512  return errors
513
514
515def main(args):
516  build_utils.InitLogging('AAB_DEBUG')
517  args = build_utils.ExpandFileArgs(args)
518  options = _ParseArgs(args)
519
520  split_dimensions = []
521  if options.split_dimensions:
522    split_dimensions = [x.upper() for x in options.split_dimensions]
523
524
525  with build_utils.TempDir() as tmp_dir:
526    logging.info('Splitting locale assets')
527    module_zips = [
528        _SplitModuleForAssetTargeting(module, tmp_dir, split_dimensions) \
529        for module in options.module_zips]
530
531    base_master_resource_ids = None
532    if options.base_module_rtxt_path:
533      logging.info('Creating R.txt allowlist')
534      base_master_resource_ids = _GenerateBaseResourcesAllowList(
535          options.base_module_rtxt_path, options.base_allowlist_rtxt_path)
536
537    logging.info('Creating BundleConfig.pb.json')
538    bundle_config = _GenerateBundleConfigJson(options.uncompressed_assets,
539                                              options.compress_dex,
540                                              split_dimensions,
541                                              base_master_resource_ids)
542
543    tmp_bundle = os.path.join(tmp_dir, 'tmp_bundle')
544
545    # Important: bundletool requires that the bundle config file is
546    # named with a .pb.json extension.
547    tmp_bundle_config = tmp_bundle + '.BundleConfig.pb.json'
548
549    with open(tmp_bundle_config, 'w') as f:
550      f.write(bundle_config)
551
552    logging.info('Running bundletool')
553    cmd_args = build_utils.JavaCmd() + [
554        '-jar',
555        bundletool.BUNDLETOOL_JAR_PATH,
556        'build-bundle',
557        '--modules=' + ','.join(module_zips),
558        '--output=' + tmp_bundle,
559        '--config=' + tmp_bundle_config,
560    ]
561
562    if options.rotation_config:
563      cmd_args += [
564          f'--metadata-file={_ROTATION_METADATA_KEY}:{options.rotation_config}'
565      ]
566
567    build_utils.CheckOutput(
568        cmd_args,
569        print_stdout=True,
570        print_stderr=True,
571        stderr_filter=build_utils.FilterReflectiveAccessJavaWarnings,
572        fail_on_output=options.warnings_as_errors)
573
574    if options.validate_services:
575      # TODO(crbug.com/1126301): This step takes 0.4s locally for bundles with
576      # isolated splits disabled and 2s for bundles with isolated splits
577      # enabled.  Consider making this run in parallel or move into a separate
578      # step before enabling isolated splits by default.
579      logging.info('Validating isolated split manifests')
580      errors = _ValidateSplits(tmp_bundle, module_zips)
581      if errors:
582        sys.stderr.write('Bundle failed sanity checks:\n  ')
583        sys.stderr.write('\n  '.join(errors))
584        sys.stderr.write('\n')
585        sys.exit(1)
586
587    logging.info('Writing final output artifacts')
588    shutil.move(tmp_bundle, options.out_bundle)
589
590  if options.rtxt_out_path:
591    _ConcatTextFiles(options.rtxt_in_paths, options.rtxt_out_path)
592
593  if options.pathmap_out_path:
594    _WriteBundlePathmap(options.pathmap_in_paths, options.module_names,
595                        options.pathmap_out_path)
596
597
598if __name__ == '__main__':
599  main(sys.argv[1:])
600