1#!/usr/bin/env python3 2# 3# Copyright 2013 The Chromium Authors 4# Use of this source code is governed by a BSD-style license that can be 5# found in the LICENSE file. 6"""Runs Android's lint tool.""" 7 8import argparse 9import logging 10import os 11import shutil 12import sys 13import time 14from xml.dom import minidom 15from xml.etree import ElementTree 16 17from util import build_utils 18from util import manifest_utils 19from util import server_utils 20import action_helpers # build_utils adds //build to sys.path. 21 22_LINT_MD_URL = 'https://chromium.googlesource.com/chromium/src/+/main/build/android/docs/lint.md' # pylint: disable=line-too-long 23 24# These checks are not useful for chromium. 25_DISABLED_ALWAYS = [ 26 "AppCompatResource", # Lint does not correctly detect our appcompat lib. 27 "Assert", # R8 --force-enable-assertions is used to enable java asserts. 28 "InflateParams", # Null is ok when inflating views for dialogs. 29 "InlinedApi", # Constants are copied so they are always available. 30 "LintBaseline", # Don't warn about using baseline.xml files. 31 "LintBaselineFixed", # We dont care if baseline.xml has unused entries. 32 "MissingInflatedId", # False positives https://crbug.com/1394222 33 "MissingApplicationIcon", # False positive for non-production targets. 34 "NetworkSecurityConfig", # Breaks on library certificates b/269783280. 35 "ObsoleteLintCustomCheck", # We have no control over custom lint checks. 36 "StringFormatCount", # Has false-positives. 37 "SwitchIntDef", # Many C++ enums are not used at all in java. 38 "Typos", # Strings are committed in English first and later translated. 39 "VisibleForTests", # Does not recognize "ForTesting" methods. 40 "UniqueConstants", # Chromium enums allow aliases. 41 "UnusedAttribute", # Chromium apks have various minSdkVersion values. 42] 43 44# These checks are not useful for test targets and adds an unnecessary burden 45# to suppress them. 46_DISABLED_FOR_TESTS = [ 47 # We should not require test strings.xml files to explicitly add 48 # translatable=false since they are not translated and not used in 49 # production. 50 "MissingTranslation", 51 # Test strings.xml files often have simple names and are not translatable, 52 # so it may conflict with a production string and cause this error. 53 "Untranslatable", 54 # Test targets often use the same strings target and resources target as the 55 # production targets but may not use all of them. 56 "UnusedResources", 57 # TODO(wnwen): Turn this back on since to crash it would require running on 58 # a device with all the various minSdkVersions. 59 # Real NewApi violations crash the app, so the only ones that lint catches 60 # but tests still succeed are false positives. 61 "NewApi", 62 # Tests should be allowed to access these methods/classes. 63 "VisibleForTests", 64] 65 66_RES_ZIP_DIR = 'RESZIPS' 67_SRCJAR_DIR = 'SRCJARS' 68_AAR_DIR = 'AARS' 69 70 71def _SrcRelative(path): 72 """Returns relative path to top-level src dir.""" 73 return os.path.relpath(path, build_utils.DIR_SOURCE_ROOT) 74 75 76def _GenerateProjectFile(android_manifest, 77 android_sdk_root, 78 cache_dir, 79 partials_dir, 80 sources=None, 81 classpath=None, 82 srcjar_sources=None, 83 resource_sources=None, 84 custom_lint_jars=None, 85 custom_annotation_zips=None, 86 android_sdk_version=None, 87 baseline_path=None): 88 project = ElementTree.Element('project') 89 root = ElementTree.SubElement(project, 'root') 90 # Run lint from output directory: crbug.com/1115594 91 root.set('dir', os.getcwd()) 92 sdk = ElementTree.SubElement(project, 'sdk') 93 # Lint requires that the sdk path be an absolute path. 94 sdk.set('dir', os.path.abspath(android_sdk_root)) 95 if baseline_path is not None: 96 baseline = ElementTree.SubElement(project, 'baseline') 97 baseline.set('file', baseline_path) 98 cache = ElementTree.SubElement(project, 'cache') 99 cache.set('dir', cache_dir) 100 main_module = ElementTree.SubElement(project, 'module') 101 main_module.set('name', 'main') 102 main_module.set('android', 'true') 103 main_module.set('library', 'false') 104 # Required to make lint-resources.xml be written to a per-target path. 105 # https://crbug.com/1515070 and b/324598620 106 main_module.set('partial-results-dir', partials_dir) 107 if android_sdk_version: 108 main_module.set('compile_sdk_version', android_sdk_version) 109 manifest = ElementTree.SubElement(main_module, 'manifest') 110 manifest.set('file', android_manifest) 111 if srcjar_sources: 112 for srcjar_file in srcjar_sources: 113 src = ElementTree.SubElement(main_module, 'src') 114 src.set('file', srcjar_file) 115 # Cannot add generated="true" since then lint does not scan them, and 116 # we get "UnusedResources" lint errors when resources are used only by 117 # generated files. 118 if sources: 119 for source in sources: 120 src = ElementTree.SubElement(main_module, 'src') 121 src.set('file', source) 122 # Cannot set test="true" since we sometimes put Test.java files beside 123 # non-test files, which lint does not allow: 124 # "Test sources cannot be in the same source root as production files" 125 if classpath: 126 for file_path in classpath: 127 classpath_element = ElementTree.SubElement(main_module, 'classpath') 128 classpath_element.set('file', file_path) 129 if resource_sources: 130 for resource_file in resource_sources: 131 resource = ElementTree.SubElement(main_module, 'resource') 132 resource.set('file', resource_file) 133 if custom_lint_jars: 134 for lint_jar in custom_lint_jars: 135 lint = ElementTree.SubElement(main_module, 'lint-checks') 136 lint.set('file', lint_jar) 137 if custom_annotation_zips: 138 for annotation_zip in custom_annotation_zips: 139 annotation = ElementTree.SubElement(main_module, 'annotations') 140 annotation.set('file', annotation_zip) 141 return project 142 143 144def _RetrieveBackportedMethods(backported_methods_path): 145 with open(backported_methods_path) as f: 146 methods = f.read().splitlines() 147 # Methods look like: 148 # java/util/Set#of(Ljava/lang/Object;)Ljava/util/Set; 149 # But error message looks like: 150 # Call requires API level R (current min is 21): java.util.Set#of [NewApi] 151 methods = (m.replace('/', '\\.') for m in methods) 152 methods = (m[:m.index('(')] for m in methods) 153 return sorted(set(methods)) 154 155 156def _GenerateConfigXmlTree(orig_config_path, backported_methods): 157 if orig_config_path: 158 root_node = ElementTree.parse(orig_config_path).getroot() 159 else: 160 root_node = ElementTree.fromstring('<lint/>') 161 162 issue_node = ElementTree.SubElement(root_node, 'issue') 163 issue_node.attrib['id'] = 'NewApi' 164 ignore_node = ElementTree.SubElement(issue_node, 'ignore') 165 ignore_node.attrib['regexp'] = '|'.join(backported_methods) 166 return root_node 167 168 169def _GenerateAndroidManifest(original_manifest_path, extra_manifest_paths, 170 min_sdk_version, android_sdk_version): 171 # Set minSdkVersion in the manifest to the correct value. 172 doc, manifest, app_node = manifest_utils.ParseManifest(original_manifest_path) 173 174 # TODO(crbug.com/1126301): Should this be done using manifest merging? 175 # Add anything in the application node of the extra manifests to the main 176 # manifest to prevent unused resource errors. 177 for path in extra_manifest_paths: 178 _, _, extra_app_node = manifest_utils.ParseManifest(path) 179 for node in extra_app_node: 180 app_node.append(node) 181 182 uses_sdk = manifest.find('./uses-sdk') 183 if uses_sdk is None: 184 uses_sdk = ElementTree.Element('uses-sdk') 185 manifest.insert(0, uses_sdk) 186 uses_sdk.set('{%s}minSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 187 min_sdk_version) 188 uses_sdk.set('{%s}targetSdkVersion' % manifest_utils.ANDROID_NAMESPACE, 189 android_sdk_version) 190 return doc 191 192 193def _WriteXmlFile(root, path): 194 logging.info('Writing xml file %s', path) 195 build_utils.MakeDirectory(os.path.dirname(path)) 196 with action_helpers.atomic_output(path) as f: 197 # Although we can write it just with ElementTree.tostring, using minidom 198 # makes it a lot easier to read as a human (also on code search). 199 f.write( 200 minidom.parseString(ElementTree.tostring( 201 root, encoding='utf-8')).toprettyxml(indent=' ').encode('utf-8')) 202 203 204def _RunLint(custom_lint_jar_path, 205 lint_jar_path, 206 backported_methods_path, 207 config_path, 208 manifest_path, 209 extra_manifest_paths, 210 sources, 211 classpath, 212 cache_dir, 213 android_sdk_version, 214 aars, 215 srcjars, 216 min_sdk_version, 217 resource_sources, 218 resource_zips, 219 android_sdk_root, 220 lint_gen_dir, 221 baseline, 222 testonly_target=False, 223 warnings_as_errors=False): 224 logging.info('Lint starting') 225 if not cache_dir: 226 # Use per-target cache directory when --cache-dir is not used. 227 cache_dir = os.path.join(lint_gen_dir, 'cache') 228 # Lint complains if the directory does not exist. 229 # When --create-cache is used, ninja will create this directory because the 230 # stamp file is created within it. 231 os.makedirs(cache_dir, exist_ok=True) 232 233 if baseline and not os.path.exists(baseline): 234 # Generating new baselines is only done locally, and requires more memory to 235 # avoid OOMs. 236 creating_baseline = True 237 lint_xmx = '4G' 238 else: 239 creating_baseline = False 240 lint_xmx = '2G' 241 242 # Lint requires this directory to exist and be cleared. 243 # See b/324598620 244 partials_dir = os.path.join(lint_gen_dir, 'partials') 245 shutil.rmtree(partials_dir, ignore_errors=True) 246 os.makedirs(partials_dir) 247 248 # All paths in lint are based off of relative paths from root with root as the 249 # prefix. Path variable substitution is based off of prefix matching so custom 250 # path variables need to match exactly in order to show up in baseline files. 251 # e.g. lint_path=path/to/output/dir/../../file/in/src 252 root_path = os.getcwd() # This is usually the output directory. 253 pathvar_src = os.path.join( 254 root_path, os.path.relpath(build_utils.DIR_SOURCE_ROOT, start=root_path)) 255 256 cmd = build_utils.JavaCmd(xmx=lint_xmx) + [ 257 '-cp', 258 '{}:{}'.format(lint_jar_path, custom_lint_jar_path), 259 'org.chromium.build.CustomLint', 260 '--sdk-home', 261 android_sdk_root, 262 '--jdk-home', 263 build_utils.JAVA_HOME, 264 '--path-variables', 265 f'SRC={pathvar_src}', 266 '--offline', 267 '--quiet', # Silences lint's "." progress updates. 268 '--stacktrace', # Prints full stacktraces for internal lint errors. 269 '--disable', 270 ','.join(_DISABLED_ALWAYS), 271 ] 272 273 if testonly_target: 274 cmd.extend(['--disable', ','.join(_DISABLED_FOR_TESTS)]) 275 276 if not manifest_path: 277 manifest_path = os.path.join(build_utils.DIR_SOURCE_ROOT, 'build', 278 'android', 'AndroidManifest.xml') 279 280 logging.info('Generating config.xml') 281 backported_methods = _RetrieveBackportedMethods(backported_methods_path) 282 config_xml_node = _GenerateConfigXmlTree(config_path, backported_methods) 283 generated_config_path = os.path.join(lint_gen_dir, 'config.xml') 284 _WriteXmlFile(config_xml_node, generated_config_path) 285 cmd.extend(['--config', generated_config_path]) 286 287 logging.info('Generating Android manifest file') 288 android_manifest_tree = _GenerateAndroidManifest(manifest_path, 289 extra_manifest_paths, 290 min_sdk_version, 291 android_sdk_version) 292 # Just use a hardcoded name, since we may have different target names (and 293 # thus different manifest_paths) using the same lint baseline. Eg. 294 # trichrome_chrome_bundle and trichrome_chrome_32_64_bundle. 295 lint_android_manifest_path = os.path.join(lint_gen_dir, 'AndroidManifest.xml') 296 _WriteXmlFile(android_manifest_tree.getroot(), lint_android_manifest_path) 297 298 resource_root_dir = os.path.join(lint_gen_dir, _RES_ZIP_DIR) 299 # These are zip files with generated resources (e. g. strings from GRD). 300 logging.info('Extracting resource zips') 301 for resource_zip in resource_zips: 302 # Use a consistent root and name rather than a temporary file so that 303 # suppressions can be local to the lint target and the resource target. 304 resource_dir = os.path.join(resource_root_dir, resource_zip) 305 shutil.rmtree(resource_dir, True) 306 os.makedirs(resource_dir) 307 resource_sources.extend( 308 build_utils.ExtractAll(resource_zip, path=resource_dir)) 309 310 logging.info('Extracting aars') 311 aar_root_dir = os.path.join(lint_gen_dir, _AAR_DIR) 312 custom_lint_jars = [] 313 custom_annotation_zips = [] 314 if aars: 315 for aar in aars: 316 # Use relative source for aar files since they are not generated. 317 aar_dir = os.path.join(aar_root_dir, 318 os.path.splitext(_SrcRelative(aar))[0]) 319 shutil.rmtree(aar_dir, True) 320 os.makedirs(aar_dir) 321 aar_files = build_utils.ExtractAll(aar, path=aar_dir) 322 for f in aar_files: 323 if f.endswith('lint.jar'): 324 custom_lint_jars.append(f) 325 elif f.endswith('annotations.zip'): 326 custom_annotation_zips.append(f) 327 328 logging.info('Extracting srcjars') 329 srcjar_root_dir = os.path.join(lint_gen_dir, _SRCJAR_DIR) 330 srcjar_sources = [] 331 if srcjars: 332 for srcjar in srcjars: 333 # Use path without extensions since otherwise the file name includes 334 # .srcjar and lint treats it as a srcjar. 335 srcjar_dir = os.path.join(srcjar_root_dir, os.path.splitext(srcjar)[0]) 336 shutil.rmtree(srcjar_dir, True) 337 os.makedirs(srcjar_dir) 338 # Sadly lint's srcjar support is broken since it only considers the first 339 # srcjar. Until we roll a lint version with that fixed, we need to extract 340 # it ourselves. 341 srcjar_sources.extend(build_utils.ExtractAll(srcjar, path=srcjar_dir)) 342 343 logging.info('Generating project file') 344 project_file_root = _GenerateProjectFile( 345 lint_android_manifest_path, android_sdk_root, cache_dir, partials_dir, 346 sources, classpath, srcjar_sources, resource_sources, custom_lint_jars, 347 custom_annotation_zips, android_sdk_version, baseline) 348 349 project_xml_path = os.path.join(lint_gen_dir, 'project.xml') 350 _WriteXmlFile(project_file_root, project_xml_path) 351 cmd += ['--project', project_xml_path] 352 353 # This filter is necessary for JDK11. 354 stderr_filter = build_utils.FilterReflectiveAccessJavaWarnings 355 stdout_filter = lambda x: build_utils.FilterLines(x, 'No issues found') 356 357 start = time.time() 358 logging.debug('Lint command %s', ' '.join(cmd)) 359 failed = False 360 361 if creating_baseline and not warnings_as_errors: 362 # Allow error code 6 when creating a baseline: ERRNO_CREATED_BASELINE 363 fail_func = lambda returncode, _: returncode not in (0, 6) 364 else: 365 fail_func = lambda returncode, _: returncode != 0 366 367 try: 368 build_utils.CheckOutput(cmd, 369 print_stdout=True, 370 stdout_filter=stdout_filter, 371 stderr_filter=stderr_filter, 372 fail_on_output=warnings_as_errors, 373 fail_func=fail_func) 374 except build_utils.CalledProcessError as e: 375 failed = True 376 # Do not output the python stacktrace because it is lengthy and is not 377 # relevant to the actual lint error. 378 sys.stderr.write(e.output) 379 finally: 380 # When not treating warnings as errors, display the extra footer. 381 is_debug = os.environ.get('LINT_DEBUG', '0') != '0' 382 383 end = time.time() - start 384 logging.info('Lint command took %ss', end) 385 if not is_debug: 386 shutil.rmtree(aar_root_dir, ignore_errors=True) 387 shutil.rmtree(resource_root_dir, ignore_errors=True) 388 shutil.rmtree(srcjar_root_dir, ignore_errors=True) 389 os.unlink(project_xml_path) 390 shutil.rmtree(partials_dir, ignore_errors=True) 391 392 if failed: 393 print('- For more help with lint in Chrome:', _LINT_MD_URL) 394 if is_debug: 395 print('- DEBUG MODE: Here is the project.xml: {}'.format( 396 _SrcRelative(project_xml_path))) 397 else: 398 print('- Run with LINT_DEBUG=1 to enable lint configuration debugging') 399 sys.exit(1) 400 401 logging.info('Lint completed') 402 403 404def _ParseArgs(argv): 405 parser = argparse.ArgumentParser() 406 action_helpers.add_depfile_arg(parser) 407 parser.add_argument('--target-name', help='Fully qualified GN target name.') 408 parser.add_argument('--skip-build-server', 409 action='store_true', 410 help='Avoid using the build server.') 411 parser.add_argument('--use-build-server', 412 action='store_true', 413 help='Always use the build server.') 414 parser.add_argument('--lint-jar-path', 415 required=True, 416 help='Path to the lint jar.') 417 parser.add_argument('--custom-lint-jar-path', 418 required=True, 419 help='Path to our custom lint jar.') 420 parser.add_argument('--backported-methods', 421 help='Path to backported methods file created by R8.') 422 parser.add_argument('--cache-dir', 423 help='Path to the directory in which the android cache ' 424 'directory tree should be stored.') 425 parser.add_argument('--config-path', help='Path to lint suppressions file.') 426 parser.add_argument('--lint-gen-dir', 427 required=True, 428 help='Path to store generated xml files.') 429 parser.add_argument('--stamp', help='Path to stamp upon success.') 430 parser.add_argument('--android-sdk-version', 431 help='Version (API level) of the Android SDK used for ' 432 'building.') 433 parser.add_argument('--min-sdk-version', 434 required=True, 435 help='Minimal SDK version to lint against.') 436 parser.add_argument('--android-sdk-root', 437 required=True, 438 help='Lint needs an explicit path to the android sdk.') 439 parser.add_argument('--testonly', 440 action='store_true', 441 help='If set, some checks like UnusedResources will be ' 442 'disabled since they are not helpful for test ' 443 'targets.') 444 parser.add_argument('--create-cache', 445 action='store_true', 446 help='Whether this invocation is just warming the cache.') 447 parser.add_argument('--warnings-as-errors', 448 action='store_true', 449 help='Treat all warnings as errors.') 450 parser.add_argument('--sources', 451 help='A list of files containing java and kotlin source ' 452 'files.') 453 parser.add_argument('--aars', help='GN list of included aars.') 454 parser.add_argument('--srcjars', help='GN list of included srcjars.') 455 parser.add_argument('--manifest-path', 456 help='Path to original AndroidManifest.xml') 457 parser.add_argument('--extra-manifest-paths', 458 action='append', 459 help='GYP-list of manifest paths to merge into the ' 460 'original AndroidManifest.xml') 461 parser.add_argument('--resource-sources', 462 default=[], 463 action='append', 464 help='GYP-list of resource sources files, similar to ' 465 'java sources files, but for resource files.') 466 parser.add_argument('--resource-zips', 467 default=[], 468 action='append', 469 help='GYP-list of resource zips, zip files of generated ' 470 'resource files.') 471 parser.add_argument('--classpath', 472 help='List of jars to add to the classpath.') 473 parser.add_argument('--baseline', 474 help='Baseline file to ignore existing errors and fail ' 475 'on new errors.') 476 477 args = parser.parse_args(build_utils.ExpandFileArgs(argv)) 478 args.sources = action_helpers.parse_gn_list(args.sources) 479 args.aars = action_helpers.parse_gn_list(args.aars) 480 args.srcjars = action_helpers.parse_gn_list(args.srcjars) 481 args.resource_sources = action_helpers.parse_gn_list(args.resource_sources) 482 args.extra_manifest_paths = action_helpers.parse_gn_list( 483 args.extra_manifest_paths) 484 args.resource_zips = action_helpers.parse_gn_list(args.resource_zips) 485 args.classpath = action_helpers.parse_gn_list(args.classpath) 486 487 if args.baseline: 488 assert os.path.basename(args.baseline) == 'lint-baseline.xml', ( 489 'The baseline file needs to be named "lint-baseline.xml" in order for ' 490 'the autoroller to find and update it whenever lint is rolled to a new ' 491 'version.') 492 493 return args 494 495 496def main(): 497 build_utils.InitLogging('LINT_DEBUG') 498 args = _ParseArgs(sys.argv[1:]) 499 500 # TODO(wnwen): Consider removing lint cache now that there are only two lint 501 # invocations. 502 # Avoid parallelizing cache creation since lint runs without the cache defeat 503 # the purpose of creating the cache in the first place. 504 if (not args.create_cache and not args.skip_build_server 505 and server_utils.MaybeRunCommand(name=args.target_name, 506 argv=sys.argv, 507 stamp_file=args.stamp, 508 force=args.use_build_server)): 509 return 510 511 sources = [] 512 for sources_file in args.sources: 513 sources.extend(build_utils.ReadSourcesList(sources_file)) 514 resource_sources = [] 515 for resource_sources_file in args.resource_sources: 516 resource_sources.extend(build_utils.ReadSourcesList(resource_sources_file)) 517 518 possible_depfile_deps = (args.srcjars + args.resource_zips + sources + 519 resource_sources + [ 520 args.baseline, 521 args.manifest_path, 522 ]) 523 depfile_deps = [p for p in possible_depfile_deps if p] 524 525 _RunLint(args.custom_lint_jar_path, 526 args.lint_jar_path, 527 args.backported_methods, 528 args.config_path, 529 args.manifest_path, 530 args.extra_manifest_paths, 531 sources, 532 args.classpath, 533 args.cache_dir, 534 args.android_sdk_version, 535 args.aars, 536 args.srcjars, 537 args.min_sdk_version, 538 resource_sources, 539 args.resource_zips, 540 args.android_sdk_root, 541 args.lint_gen_dir, 542 args.baseline, 543 testonly_target=args.testonly, 544 warnings_as_errors=args.warnings_as_errors) 545 logging.info('Creating stamp file') 546 build_utils.Touch(args.stamp) 547 548 if args.depfile: 549 action_helpers.write_depfile(args.depfile, args.stamp, depfile_deps) 550 551 552if __name__ == '__main__': 553 sys.exit(main()) 554