1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Runs Android's lint tool.""" 7 8import argparse 9import logging 10import os 11import shutil 12import sys 13import time 14from xml.dom import minidom 15from xml.etree import ElementTree 16 17from util import build_utils 18from util import manifest_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long 23 24# These checks are not useful for chromium. 25_DISABLED_ALWAYS = [ 26 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 27 "AppLinkUrlError", # As a browser, we have intent filters without a host. 28 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 29 "InflateParams", # Null is ok when inflating views for dialogs. 30 "InlinedApi", # Constants are copied so they are always available. 31 "LintBaseline", # Don't warn about using baseline.xml files. 32 "LintBaselineFixed", # We dont care if baseline.xml has unused entries. 33 "MissingInflatedId", # False positives https://crbug.com/1394222 34 "MissingApplicationIcon", # False positive for non-production targets. 35 "MissingVersion", # We set versions via aapt2, which runs after lint. 36 "NetworkSecurityConfig", # Breaks on library certificates b/269783280. 37 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 38 "OldTargetApi", # We sometimes need targetSdkVersion to not be latest. 39 "StringFormatCount", # Has false-positives. 40 "SwitchIntDef", # Many C++ enums are not used at all in java. 41 "Typos", # Strings are committed in English first and later translated. 42 "VisibleForTests", # Does not recognize "ForTesting" methods. 43 "UniqueConstants", # Chromium enums allow aliases. 44 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 45 "NullSafeMutableLiveData", # Broken. See b/370586513. 46] 47 48_RES_ZIP_DIR = 'RESZIPS' 49_SRCJAR_DIR = 'SRCJARS' 50_AAR_DIR = 'AARS' 51 52 53def _SrcRelative(path): 54 """Returns relative path to top-level src dir.""" 55 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 56 57 58def _GenerateProjectFile(android_manifest, 59 android_sdk_root, 60 cache_dir, 61 partials_dir, 62 sources=None, 63 classpath=None, 64 srcjar_sources=None, 65 resource_sources=None, 66 custom_lint_jars=None, 67 custom_annotation_zips=None, 68 android_sdk_version=None, 69 baseline_path=None): 70 project = ElementTree.Element('project') 71 root = ElementTree.SubElement(project, 'root') 72 # Run lint from output directory: crbug.com/1115594 73 root.set('dir', os.getcwd()) 74 sdk = ElementTree.SubElement(project, 'sdk') 75 # Lint requires that the sdk path be an absolute path. 76 sdk.set('dir', os.path.abspath(android_sdk_root)) 77 if baseline_path is not None: 78 baseline = ElementTree.SubElement(project, 'baseline') 79 baseline.set('file', baseline_path) 80 cache = ElementTree.SubElement(project, 'cache') 81 cache.set('dir', cache_dir) 82 main_module = ElementTree.SubElement(project, 'module') 83 main_module.set('name', 'main') 84 main_module.set('android', 'true') 85 main_module.set('library', 'false') 86 # Required to make lint-resources.xml be written to a per-target path. 87 # https://crbug.com/1515070 and b/324598620 88 main_module.set('partial-results-dir', partials_dir) 89 if android_sdk_version: 90 main_module.set('compile_sdk_version', android_sdk_version) 91 manifest = ElementTree.SubElement(main_module, 'manifest') 92 manifest.set('file', android_manifest) 93 if srcjar_sources: 94 for srcjar_file in srcjar_sources: 95 src = ElementTree.SubElement(main_module, 'src') 96 src.set('file', srcjar_file) 97 # Cannot add generated="true" since then lint does not scan them, and 98 # we get "UnusedResources" lint errors when resources are used only by 99 # generated files. 100 if sources: 101 for source in sources: 102 src = ElementTree.SubElement(main_module, 'src') 103 src.set('file', source) 104 # Cannot set test="true" since we sometimes put Test.java files beside 105 # non-test files, which lint does not allow: 106 # "Test sources cannot be in the same source root as production files" 107 if classpath: 108 for file_path in classpath: 109 classpath_element = ElementTree.SubElement(main_module, 'classpath') 110 classpath_element.set('file', file_path) 111 if resource_sources: 112 for resource_file in resource_sources: 113 resource = ElementTree.SubElement(main_module, 'resource') 114 resource.set('file', resource_file) 115 if custom_lint_jars: 116 for lint_jar in custom_lint_jars: 117 lint = ElementTree.SubElement(main_module, 'lint-checks') 118 lint.set('file', lint_jar) 119 if custom_annotation_zips: 120 for annotation_zip in custom_annotation_zips: 121 annotation = ElementTree.SubElement(main_module, 'annotations') 122 annotation.set('file', annotation_zip) 123 return project 124 125 126def _RetrieveBackportedMethods(backported_methods_path): 127 with open(backported_methods_path) as f: 128 methods = f.read().splitlines() 129 # Methods look like: 130 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 131 # But error message looks like: 132 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 133 methods = (m.replace('/', '\\.') for m in methods) 134 methods = (m[:m.index('(')] for m in methods) 135 return sorted(set(methods)) 136 137 138def _GenerateConfigXmlTree(orig_config_path, backported_methods): 139 if orig_config_path: 140 root_node = ElementTree.parse(orig_config_path).getroot() 141 else: 142 root_node = ElementTree.fromstring('<lint/>') 143 144 issue_node = ElementTree.SubElement(root_node, 'issue') 145 issue_node.attrib['id'] = 'NewApi' 146 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 147 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 148 return root_node 149 150 151def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 152 min_sdk_version, android_sdk_version): 153 # Set minSdkVersion in the manifest to the correct value. 154 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 155 156 # TODO(crbug.com/40148088): Should this be done using manifest merging? 157 # Add anything in the application node of the extra manifests to the main 158 # manifest to prevent unused resource errors. 159 for path in extra_manifest_paths: 160 _, _, extra_app_node = manifest_utils.ParseManifest(path) 161 for node in extra_app_node: 162 app_node.append(node) 163 164 uses_sdk = manifest.find('./uses-sdk') 165 if uses_sdk is None: 166 uses_sdk = ElementTree.Element('uses-sdk') 167 manifest.insert(0, uses_sdk) 168 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 169 min_sdk_version) 170 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 171 android_sdk_version) 172 return doc 173 174 175def _WriteXmlFile(root, path): 176 logging.info('Writing xml file %s', path) 177 build_utils.MakeDirectory(os.path.dirname(path)) 178 with action_helpers.atomic_output(path) as f: 179 # Although we can write it just with ElementTree.tostring, using minidom 180 # makes it a lot easier to read as a human (also on code search). 181 f.write( 182 minidom.parseString(ElementTree.tostring( 183 root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) 184 185 186def _RunLint(custom_lint_jar_path, 187 lint_jar_path, 188 backported_methods_path, 189 config_path, 190 manifest_path, 191 extra_manifest_paths, 192 sources, 193 classpath, 194 cache_dir, 195 android_sdk_version, 196 aars, 197 srcjars, 198 min_sdk_version, 199 resource_sources, 200 resource_zips, 201 android_sdk_root, 202 lint_gen_dir, 203 baseline, 204 create_cache, 205 warnings_as_errors=False): 206 logging.info('Lint starting') 207 if not cache_dir: 208 # Use per-target cache directory when --cache-dir is not used. 209 cache_dir = os.path.join(lint_gen_dir, 'cache') 210 # Lint complains if the directory does not exist. 211 # When --create-cache is used, ninja will create this directory because the 212 # stamp file is created within it. 213 os.makedirs(cache_dir, exist_ok=True) 214 215 if baseline and not os.path.exists(baseline): 216 # Generating new baselines is only done locally, and requires more memory to 217 # avoid OOMs. 218 creating_baseline = True 219 lint_xmx = '4G' 220 else: 221 creating_baseline = False 222 lint_xmx = '3G' 223 224 # Lint requires this directory to exist and be cleared. 225 # See b/324598620 226 partials_dir = os.path.join(lint_gen_dir, 'partials') 227 shutil.rmtree(partials_dir, ignore_errors=True) 228 os.makedirs(partials_dir) 229 230 # All paths in lint are based off of relative paths from root with root as the 231 # prefix. Path variable substitution is based off of prefix matching so custom 232 # path variables need to match exactly in order to show up in baseline files. 233 # e.g. lint_path=path/to/output/dir/../../file/in/src 234 root_path = os.getcwd() # This is usually the output directory. 235 pathvar_src = os.path.join( 236 root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) 237 238 cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ 239 '-cp', 240 '{}:{}'.format(lint_jar_path, custom_lint_jar_path), 241 'org.chromium.build.CustomLint', 242 '--sdk-home', 243 android_sdk_root, 244 '--jdk-home', 245 build_utils.JAVA_HOME, 246 '--path-variables', 247 f'SRC={pathvar_src}', 248 '--offline', 249 '--quiet', # Silences lint's "." progress updates. 250 '--stacktrace', # Prints full stacktraces for internal lint errors. 251 ] 252 253 # Only disable for real runs since otherwise you get UnknownIssueId warnings 254 # when disabling custom lint checks since they are not passed during cache 255 # creation. 256 if not create_cache: 257 cmd += [ 258 '--disable', 259 ','.join(_DISABLED_ALWAYS), 260 ] 261 262 if not manifest_path: 263 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 264 'android', 'AndroidManifest.xml') 265 266 logging.info('Generating config.xml') 267 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 268 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 269 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 270 _WriteXmlFile(config_xml_node, generated_config_path) 271 cmd.extend(['--config', generated_config_path]) 272 273 logging.info('Generating Android manifest file') 274 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 275 extra_manifest_paths, 276 min_sdk_version, 277 android_sdk_version) 278 # Just use a hardcoded name, since we may have different target names (and 279 # thus different manifest_paths) using the same lint baseline. Eg. 280 # trichrome_chrome_bundle and trichrome_chrome_32_64_bundle. 281 lint_android_manifest_path = os.path.join(lint_gen_dir, 'AndroidManifest.xml') 282 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 283 284 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 285 # These are zip files with generated resources (e. g. strings from GRD). 286 logging.info('Extracting resource zips') 287 for resource_zip in resource_zips: 288 # Use a consistent root and name rather than a temporary file so that 289 # suppressions can be local to the lint target and the resource target. 290 resource_dir = os.path.join(resource_root_dir, resource_zip) 291 shutil.rmtree(resource_dir, True) 292 os.makedirs(resource_dir) 293 resource_sources.extend( 294 build_utils.ExtractAll(resource_zip, path=resource_dir)) 295 296 logging.info('Extracting aars') 297 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 298 custom_lint_jars = [] 299 custom_annotation_zips = [] 300 if aars: 301 for aar in aars: 302 # Use relative source for aar files since they are not generated. 303 aar_dir = os.path.join(aar_root_dir, 304 os.path.splitext(_SrcRelative(aar))[0]) 305 shutil.rmtree(aar_dir, True) 306 os.makedirs(aar_dir) 307 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 308 for f in aar_files: 309 if f.endswith('lint.jar'): 310 custom_lint_jars.append(f) 311 elif f.endswith('annotations.zip'): 312 custom_annotation_zips.append(f) 313 314 logging.info('Extracting srcjars') 315 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 316 srcjar_sources = [] 317 if srcjars: 318 for srcjar in srcjars: 319 # Use path without extensions since otherwise the file name includes 320 # .srcjar and lint treats it as a srcjar. 321 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 322 shutil.rmtree(srcjar_dir, True) 323 os.makedirs(srcjar_dir) 324 # Sadly lint's srcjar support is broken since it only considers the first 325 # srcjar. Until we roll a lint version with that fixed, we need to extract 326 # it ourselves. 327 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 328 329 logging.info('Generating project file') 330 project_file_root = _GenerateProjectFile( 331 lint_android_manifest_path, android_sdk_root, cache_dir, partials_dir, 332 sources, classpath, srcjar_sources, resource_sources, custom_lint_jars, 333 custom_annotation_zips, android_sdk_version, baseline) 334 335 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 336 _WriteXmlFile(project_file_root, project_xml_path) 337 cmd += ['--project', project_xml_path] 338 339 def stdout_filter(output): 340 filter_patterns = [ 341 # This filter is necessary for JDK11. 342 'No issues found', 343 # Custom checks are not always available in every lint run so an 344 # UnknownIssueId warning is sometimes printed for custom checks in the 345 # _DISABLED_ALWAYS list. 346 r'\[UnknownIssueId\]', 347 # If all the warnings are filtered, we should not fail on the final 348 # summary line. 349 r'\d+ errors, \d+ warnings', 350 ] 351 return build_utils.FilterLines(output, '|'.join(filter_patterns)) 352 353 def stderr_filter(output): 354 output = build_utils.FilterReflectiveAccessJavaWarnings(output) 355 # Presumably a side-effect of our manual manifest merging, but does not 356 # seem to actually break anything: 357 # "Manifest merger failed with multiple errors, see logs" 358 return build_utils.FilterLines(output, 'Manifest merger failed') 359 360 start = time.time() 361 logging.debug('Lint command %s', ' '.join(cmd)) 362 failed = False 363 364 if creating_baseline and not warnings_as_errors: 365 # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE 366 fail_func = lambda returncode, _: returncode not in (0, 6) 367 else: 368 fail_func = lambda returncode, _: returncode != 0 369 370 try: 371 build_utils.CheckOutput(cmd, 372 print_stdout=True, 373 stdout_filter=stdout_filter, 374 stderr_filter=stderr_filter, 375 fail_on_output=warnings_as_errors, 376 fail_func=fail_func) 377 except build_utils.CalledProcessError as e: 378 failed = True 379 # Do not output the python stacktrace because it is lengthy and is not 380 # relevant to the actual lint error. 381 sys.stderr.write(e.output) 382 finally: 383 # When not treating warnings as errors, display the extra footer. 384 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 385 386 end = time.time() - start 387 logging.info('Lint command took %ss', end) 388 if not is_debug: 389 shutil.rmtree(aar_root_dir, ignore_errors=True) 390 shutil.rmtree(resource_root_dir, ignore_errors=True) 391 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 392 os.unlink(project_xml_path) 393 shutil.rmtree(partials_dir, ignore_errors=True) 394 395 if failed: 396 print('- For more help with lint in Chrome:', _LINT_MD_URL) 397 if is_debug: 398 print('- DEBUG MODE: Here is the project.xml: {}'.format( 399 _SrcRelative(project_xml_path))) 400 else: 401 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 402 sys.exit(1) 403 404 logging.info('Lint completed') 405 406 407def _ParseArgs(argv): 408 parser = argparse.ArgumentParser() 409 action_helpers.add_depfile_arg(parser) 410 parser.add_argument('--target-name', help='Fully qualified GN target name.') 411 parser.add_argument('--skip-build-server', 412 action='store_true', 413 help='Avoid using the build server.') 414 parser.add_argument('--use-build-server', 415 action='store_true', 416 help='Always use the build server.') 417 parser.add_argument('--experimental-build-server', 418 action='store_true', 419 help='Use experimental build server features.') 420 parser.add_argument('--lint-jar-path', 421 required=True, 422 help='Path to the lint jar.') 423 parser.add_argument('--custom-lint-jar-path', 424 required=True, 425 help='Path to our custom lint jar.') 426 parser.add_argument('--backported-methods', 427 help='Path to backported methods file created by R8.') 428 parser.add_argument('--cache-dir', 429 help='Path to the directory in which the android cache ' 430 'directory tree should be stored.') 431 parser.add_argument('--config-path', help='Path to lint suppressions file.') 432 parser.add_argument('--lint-gen-dir', 433 required=True, 434 help='Path to store generated xml files.') 435 parser.add_argument('--stamp', help='Path to stamp upon success.') 436 parser.add_argument('--android-sdk-version', 437 help='Version (API level) of the Android SDK used for ' 438 'building.') 439 parser.add_argument('--min-sdk-version', 440 required=True, 441 help='Minimal SDK version to lint against.') 442 parser.add_argument('--android-sdk-root', 443 required=True, 444 help='Lint needs an explicit path to the android sdk.') 445 parser.add_argument('--create-cache', 446 action='store_true', 447 help='Whether this invocation is just warming the cache.') 448 parser.add_argument('--warnings-as-errors', 449 action='store_true', 450 help='Treat all warnings as errors.') 451 parser.add_argument('--sources', 452 help='A list of files containing java and kotlin source ' 453 'files.') 454 parser.add_argument('--aars', help='GN list of included aars.') 455 parser.add_argument('--srcjars', help='GN list of included srcjars.') 456 parser.add_argument('--manifest-path', 457 help='Path to original AndroidManifest.xml') 458 parser.add_argument('--extra-manifest-paths', 459 action='append', 460 help='GYP-list of manifest paths to merge into the ' 461 'original AndroidManifest.xml') 462 parser.add_argument('--resource-sources', 463 default=[], 464 action='append', 465 help='GYP-list of resource sources files, similar to ' 466 'java sources files, but for resource files.') 467 parser.add_argument('--resource-zips', 468 default=[], 469 action='append', 470 help='GYP-list of resource zips, zip files of generated ' 471 'resource files.') 472 parser.add_argument('--classpath', 473 help='List of jars to add to the classpath.') 474 parser.add_argument('--baseline', 475 help='Baseline file to ignore existing errors and fail ' 476 'on new errors.') 477 478 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 479 args.sources = action_helpers.parse_gn_list(args.sources) 480 args.aars = action_helpers.parse_gn_list(args.aars) 481 args.srcjars = action_helpers.parse_gn_list(args.srcjars) 482 args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) 483 args.extra_manifest_paths = action_helpers.parse_gn_list( 484 args.extra_manifest_paths) 485 args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) 486 args.classpath = action_helpers.parse_gn_list(args.classpath) 487 488 if args.baseline: 489 assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( 490 'The baseline file needs to be named "lint-baseline.xml" in order for ' 491 'the autoroller to find and update it whenever lint is rolled to a new ' 492 'version.') 493 494 return args 495 496 497def main(): 498 build_utils.InitLogging('LINT_DEBUG') 499 args = _ParseArgs(sys.argv[1:]) 500 501 sources = [] 502 for sources_file in args.sources: 503 sources.extend(build_utils.ReadSourcesList(sources_file)) 504 resource_sources = [] 505 for resource_sources_file in args.resource_sources: 506 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 507 508 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 509 resource_sources + [ 510 args.baseline, 511 args.manifest_path, 512 ]) 513 depfile_deps = [p for p in possible_depfile_deps if p] 514 515 if args.depfile: 516 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 517 518 # TODO(wnwen): Consider removing lint cache now that there are only two lint 519 # invocations. 520 # Avoid parallelizing cache creation since lint runs without the cache defeat 521 # the purpose of creating the cache in the first place. Forward the command 522 # after the depfile has been written as siso requires it. 523 if (not args.create_cache and not args.skip_build_server 524 and server_utils.MaybeRunCommand( 525 name=args.target_name, 526 argv=sys.argv, 527 stamp_file=args.stamp, 528 force=args.use_build_server, 529 experimental=args.experimental_build_server)): 530 return 531 532 _RunLint(args.custom_lint_jar_path, 533 args.lint_jar_path, 534 args.backported_methods, 535 args.config_path, 536 args.manifest_path, 537 args.extra_manifest_paths, 538 sources, 539 args.classpath, 540 args.cache_dir, 541 args.android_sdk_version, 542 args.aars, 543 args.srcjars, 544 args.min_sdk_version, 545 resource_sources, 546 args.resource_zips, 547 args.android_sdk_root, 548 args.lint_gen_dir, 549 args.baseline, 550 args.create_cache, 551 warnings_as_errors=args.warnings_as_errors) 552 logging.info('Creating stamp file') 553 build_utils.Touch(args.stamp) 554 555 556if __name__ == '__main__': 557 sys.exit(main()) 558