1# Copyright 2016 The Android Open Source Project 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14 15"""Functions that implement the actual checks.""" 16 17import fnmatch 18import json 19import os 20import platform 21import re 22import sys 23from typing import Callable, NamedTuple 24 25_path = os.path.realpath(__file__ + '/../..') 26if sys.path[0] != _path: 27 sys.path.insert(0, _path) 28del _path 29 30# pylint: disable=wrong-import-position 31import rh.git 32import rh.results 33import rh.utils 34 35 36class Placeholders(object): 37 """Holder class for replacing ${vars} in arg lists. 38 39 To add a new variable to replace in config files, just add it as a @property 40 to this class using the form. So to add support for BIRD: 41 @property 42 def var_BIRD(self): 43 return <whatever this is> 44 45 You can return either a string or an iterable (e.g. a list or tuple). 46 """ 47 48 def __init__(self, diff=()): 49 """Initialize. 50 51 Args: 52 diff: The list of files that changed. 53 """ 54 self.diff = diff 55 56 def expand_vars(self, args): 57 """Perform place holder expansion on all of |args|. 58 59 Args: 60 args: The args to perform expansion on. 61 62 Returns: 63 The updated |args| list. 64 """ 65 all_vars = set(self.vars()) 66 replacements = dict((var, self.get(var)) for var in all_vars) 67 68 ret = [] 69 for arg in args: 70 if arg.endswith('${PREUPLOAD_FILES_PREFIXED}'): 71 if arg == '${PREUPLOAD_FILES_PREFIXED}': 72 assert len(ret) > 1, ('PREUPLOAD_FILES_PREFIXED cannot be ' 73 'the 1st or 2nd argument') 74 prev_arg = ret[-1] 75 ret = ret[0:-1] 76 for file in self.get('PREUPLOAD_FILES'): 77 ret.append(prev_arg) 78 ret.append(file) 79 else: 80 prefix = arg[0:-len('${PREUPLOAD_FILES_PREFIXED}')] 81 ret.extend( 82 prefix + file for file in self.get('PREUPLOAD_FILES')) 83 else: 84 # First scan for exact matches 85 for key, val in replacements.items(): 86 var = '${' + key + '}' 87 if arg == var: 88 if isinstance(val, str): 89 ret.append(val) 90 else: 91 ret.extend(val) 92 # We break on first hit to avoid double expansion. 93 break 94 else: 95 # If no exact matches, do an inline replacement. 96 def replace(m): 97 val = self.get(m.group(1)) 98 if isinstance(val, str): 99 return val 100 return ' '.join(val) 101 ret.append(re.sub(r'\$\{(' + '|'.join(all_vars) + r')\}', 102 replace, arg)) 103 return ret 104 105 @classmethod 106 def vars(cls): 107 """Yield all replacement variable names.""" 108 for key in dir(cls): 109 if key.startswith('var_'): 110 yield key[4:] 111 112 def get(self, var): 113 """Helper function to get the replacement |var| value.""" 114 return getattr(self, f'var_{var}') 115 116 @property 117 def var_PREUPLOAD_COMMIT_MESSAGE(self): 118 """The git commit message.""" 119 return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '') 120 121 @property 122 def var_PREUPLOAD_COMMIT(self): 123 """The git commit sha1.""" 124 return os.environ.get('PREUPLOAD_COMMIT', '') 125 126 @property 127 def var_PREUPLOAD_FILES(self): 128 """List of files modified in this git commit.""" 129 return [x.file for x in self.diff if x.status != 'D'] 130 131 @property 132 def var_REPO_PATH(self): 133 """The path to the project relative to the root""" 134 return os.environ.get('REPO_PATH', '') 135 136 @property 137 def var_REPO_PROJECT(self): 138 """The name of the project""" 139 return os.environ.get('REPO_PROJECT', '') 140 141 @property 142 def var_REPO_ROOT(self): 143 """The root of the repo (sub-manifest) checkout.""" 144 return rh.git.find_repo_root() 145 146 @property 147 def var_REPO_OUTER_ROOT(self): 148 """The root of the repo (outer) checkout.""" 149 return rh.git.find_repo_root(outer=True) 150 151 @property 152 def var_BUILD_OS(self): 153 """The build OS (see _get_build_os_name for details).""" 154 return _get_build_os_name() 155 156 157class ExclusionScope(object): 158 """Exclusion scope for a hook. 159 160 An exclusion scope can be used to determine if a hook has been disabled for 161 a specific project. 162 """ 163 164 def __init__(self, scope): 165 """Initialize. 166 167 Args: 168 scope: A list of shell-style wildcards (fnmatch) or regular 169 expression. Regular expressions must start with the ^ character. 170 """ 171 self._scope = [] 172 for path in scope: 173 if path.startswith('^'): 174 self._scope.append(re.compile(path)) 175 else: 176 self._scope.append(path) 177 178 def __contains__(self, proj_dir): 179 """Checks if |proj_dir| matches the excluded paths. 180 181 Args: 182 proj_dir: The relative path of the project. 183 """ 184 for exclusion_path in self._scope: 185 if hasattr(exclusion_path, 'match'): 186 if exclusion_path.match(proj_dir): 187 return True 188 elif fnmatch.fnmatch(proj_dir, exclusion_path): 189 return True 190 return False 191 192 193class HookOptions(object): 194 """Holder class for hook options.""" 195 196 def __init__(self, name, args, tool_paths): 197 """Initialize. 198 199 Args: 200 name: The name of the hook. 201 args: The override commandline arguments for the hook. 202 tool_paths: A dictionary with tool names to paths. 203 """ 204 self.name = name 205 self._args = args 206 self._tool_paths = tool_paths 207 208 @staticmethod 209 def expand_vars(args, diff=()): 210 """Perform place holder expansion on all of |args|.""" 211 replacer = Placeholders(diff=diff) 212 return replacer.expand_vars(args) 213 214 def args(self, default_args=(), diff=()): 215 """Gets the hook arguments, after performing place holder expansion. 216 217 Args: 218 default_args: The list to return if |self._args| is empty. 219 diff: The list of files that changed in the current commit. 220 221 Returns: 222 A list with arguments. 223 """ 224 args = self._args 225 if not args: 226 args = default_args 227 228 return self.expand_vars(args, diff=diff) 229 230 def tool_path(self, tool_name): 231 """Gets the path in which the |tool_name| executable can be found. 232 233 This function performs expansion for some place holders. If the tool 234 does not exist in the overridden |self._tool_paths| dictionary, the tool 235 name will be returned and will be run from the user's $PATH. 236 237 Args: 238 tool_name: The name of the executable. 239 240 Returns: 241 The path of the tool with all optional place holders expanded. 242 """ 243 assert tool_name in TOOL_PATHS 244 if tool_name not in self._tool_paths: 245 return TOOL_PATHS[tool_name] 246 247 tool_path = os.path.normpath(self._tool_paths[tool_name]) 248 return self.expand_vars([tool_path])[0] 249 250 251class CallableHook(NamedTuple): 252 """A callable hook.""" 253 name: str 254 hook: Callable 255 scope: ExclusionScope 256 257 258def _run(cmd, **kwargs): 259 """Helper command for checks that tend to gather output.""" 260 kwargs.setdefault('combine_stdout_stderr', True) 261 kwargs.setdefault('capture_output', True) 262 kwargs.setdefault('check', False) 263 # Make sure hooks run with stdin disconnected to avoid accidentally 264 # interactive tools causing pauses. 265 kwargs.setdefault('input', '') 266 return rh.utils.run(cmd, **kwargs) 267 268 269def _match_regex_list(subject, expressions): 270 """Try to match a list of regular expressions to a string. 271 272 Args: 273 subject: The string to match regexes on. 274 expressions: An iterable of regular expressions to check for matches with. 275 276 Returns: 277 Whether the passed in subject matches any of the passed in regexes. 278 """ 279 for expr in expressions: 280 if re.search(expr, subject): 281 return True 282 return False 283 284 285def _filter_diff(diff, include_list, exclude_list=()): 286 """Filter out files based on the conditions passed in. 287 288 Args: 289 diff: list of diff objects to filter. 290 include_list: list of regex that when matched with a file path will cause 291 it to be added to the output list unless the file is also matched with 292 a regex in the exclude_list. 293 exclude_list: list of regex that when matched with a file will prevent it 294 from being added to the output list, even if it is also matched with a 295 regex in the include_list. 296 297 Returns: 298 A list of filepaths that contain files matched in the include_list and not 299 in the exclude_list. 300 """ 301 filtered = [] 302 for d in diff: 303 if (d.status != 'D' and 304 _match_regex_list(d.file, include_list) and 305 not _match_regex_list(d.file, exclude_list)): 306 # We've got a match! 307 filtered.append(d) 308 return filtered 309 310 311def _get_build_os_name(): 312 """Gets the build OS name. 313 314 Returns: 315 A string in a format usable to get prebuilt tool paths. 316 """ 317 system = platform.system() 318 if 'Darwin' in system or 'Macintosh' in system: 319 return 'darwin-x86' 320 321 # TODO: Add more values if needed. 322 return 'linux-x86' 323 324 325def _check_cmd(hook_name, project, commit, cmd, fixup_cmd=None, **kwargs): 326 """Runs |cmd| and returns its result as a HookCommandResult.""" 327 return [rh.results.HookCommandResult(hook_name, project, commit, 328 _run(cmd, **kwargs), 329 fixup_cmd=fixup_cmd)] 330 331 332# Where helper programs exist. 333TOOLS_DIR = os.path.realpath(__file__ + '/../../tools') 334 335def get_helper_path(tool): 336 """Return the full path to the helper |tool|.""" 337 return os.path.join(TOOLS_DIR, tool) 338 339 340def check_custom(project, commit, _desc, diff, options=None, **kwargs): 341 """Run a custom hook.""" 342 return _check_cmd(options.name, project, commit, options.args((), diff), 343 **kwargs) 344 345 346def check_aosp_license(project, commit, _desc, diff, options=None): 347 """Checks that if all new added files has AOSP licenses""" 348 349 exclude_dir_args = [x for x in options.args() 350 if x.startswith('--exclude-dirs=')] 351 exclude_dirs = [x[len('--exclude-dirs='):].split(',') 352 for x in exclude_dir_args] 353 exclude_list = [fr'^{x}/.*$' for dir_list in exclude_dirs for x in dir_list] 354 355 # Filter diff based on extension. 356 include_list = [ 357 # Coding languages and scripts. 358 r".*\.c$", 359 r".*\.cc$", 360 r".*\.cpp$", 361 r".*\.h$", 362 r".*\.java$", 363 r".*\.kt$", 364 r".*\.rs$", 365 r".*\.py$", 366 r".*\.sh$", 367 368 # Build and config files. 369 r".*\.bp$", 370 r".*\.mk$", 371 r".*\.xml$", 372 ] 373 diff = _filter_diff(diff, include_list, exclude_list) 374 375 # Only check the new-added files. 376 diff = [d for d in diff if d.status == 'A'] 377 378 if not diff: 379 return None 380 381 cmd = [get_helper_path('check_aosp_license.py'), '--commit_hash', commit] 382 cmd += HookOptions.expand_vars(('${PREUPLOAD_FILES}',), diff) 383 return _check_cmd('aosp_license', project, commit, cmd) 384 385 386def check_bpfmt(project, commit, _desc, diff, options=None): 387 """Checks that Blueprint files are formatted with bpfmt.""" 388 filtered = _filter_diff(diff, [r'\.bp$']) 389 if not filtered: 390 return None 391 392 bpfmt = options.tool_path('bpfmt') 393 bpfmt_options = options.args((), filtered) 394 cmd = [bpfmt, '-d'] + bpfmt_options 395 fixup_cmd = [bpfmt, '-w'] 396 if '-s' in bpfmt_options: 397 fixup_cmd.append('-s') 398 fixup_cmd.append('--') 399 400 ret = [] 401 for d in filtered: 402 data = rh.git.get_file_content(commit, d.file) 403 result = _run(cmd, input=data) 404 if result.stdout: 405 ret.append(rh.results.HookResult( 406 'bpfmt', project, commit, 407 error=result.stdout, 408 files=(d.file,), 409 fixup_cmd=fixup_cmd)) 410 return ret 411 412 413def check_checkpatch(project, commit, _desc, diff, options=None): 414 """Run |diff| through the kernel's checkpatch.pl tool.""" 415 tool = get_helper_path('checkpatch.pl') 416 cmd = ([tool, '-', '--root', project.dir] + 417 options.args(('--ignore=GERRIT_CHANGE_ID',), diff)) 418 return _check_cmd('checkpatch.pl', project, commit, cmd, 419 input=rh.git.get_patch(commit)) 420 421 422def check_clang_format(project, commit, _desc, diff, options=None): 423 """Run git clang-format on the commit.""" 424 tool = get_helper_path('clang-format.py') 425 clang_format = options.tool_path('clang-format') 426 git_clang_format = options.tool_path('git-clang-format') 427 tool_args = (['--clang-format', clang_format, '--git-clang-format', 428 git_clang_format] + 429 options.args(('--style', 'file', '--commit', commit), diff)) 430 cmd = [tool] + tool_args 431 fixup_cmd = [tool, '--fix'] + tool_args 432 return _check_cmd('clang-format', project, commit, cmd, 433 fixup_cmd=fixup_cmd) 434 435 436def check_google_java_format(project, commit, _desc, _diff, options=None): 437 """Run google-java-format on the commit.""" 438 include_dir_args = [x for x in options.args() 439 if x.startswith('--include-dirs=')] 440 include_dirs = [x[len('--include-dirs='):].split(',') 441 for x in include_dir_args] 442 patterns = [fr'^{x}/.*\.java$' for dir_list in include_dirs 443 for x in dir_list] 444 if not patterns: 445 patterns = [r'\.java$'] 446 447 filtered = _filter_diff(_diff, patterns) 448 449 if not filtered: 450 return None 451 452 args = [x for x in options.args() if x not in include_dir_args] 453 454 tool = get_helper_path('google-java-format.py') 455 google_java_format = options.tool_path('google-java-format') 456 google_java_format_diff = options.tool_path('google-java-format-diff') 457 tool_args = ['--google-java-format', google_java_format, 458 '--google-java-format-diff', google_java_format_diff, 459 '--commit', commit] + args 460 cmd = [tool] + tool_args + HookOptions.expand_vars( 461 ('${PREUPLOAD_FILES}',), filtered) 462 fixup_cmd = [tool, '--fix'] + tool_args 463 return [rh.results.HookCommandResult('google-java-format', project, commit, 464 _run(cmd), 465 files=[x.file for x in filtered], 466 fixup_cmd=fixup_cmd)] 467 468 469def check_ktfmt(project, commit, _desc, diff, options=None): 470 """Checks that kotlin files are formatted with ktfmt.""" 471 472 include_dir_args = [x for x in options.args() 473 if x.startswith('--include-dirs=')] 474 include_dirs = [x[len('--include-dirs='):].split(',') 475 for x in include_dir_args] 476 patterns = [fr'^{x}/.*\.kt$' for dir_list in include_dirs 477 for x in dir_list] 478 if not patterns: 479 patterns = [r'\.kt$'] 480 481 filtered = _filter_diff(diff, patterns) 482 483 if not filtered: 484 return None 485 486 args = [x for x in options.args() if x not in include_dir_args] 487 488 ktfmt = options.tool_path('ktfmt') 489 cmd = [ktfmt, '--dry-run'] + args + HookOptions.expand_vars( 490 ('${PREUPLOAD_FILES}',), filtered) 491 result = _run(cmd) 492 if result.stdout: 493 fixup_cmd = [ktfmt] + args 494 return [rh.results.HookResult( 495 'ktfmt', project, commit, error='Formatting errors detected', 496 files=[x.file for x in filtered], fixup_cmd=fixup_cmd)] 497 return None 498 499 500def check_commit_msg_bug_field(project, commit, desc, _diff, options=None): 501 """Check the commit message for a 'Bug:' or 'Fix:' line.""" 502 regex = r'^(Bug|Fix): (None|[0-9]+(, [0-9]+)*)$' 503 check_re = re.compile(regex) 504 505 if options.args(): 506 raise ValueError('commit msg Bug check takes no options') 507 508 found = [] 509 for line in desc.splitlines(): 510 if check_re.match(line): 511 found.append(line) 512 513 if not found: 514 error = ( 515 'Commit message is missing a "Bug:" line. It must match the\n' 516 f'following case-sensitive regex:\n\n {regex}' 517 ) 518 else: 519 return None 520 521 return [rh.results.HookResult('commit msg: "Bug:" check', 522 project, commit, error=error)] 523 524 525def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None): 526 """Check the commit message for a 'Change-Id:' line.""" 527 field = 'Change-Id' 528 regex = fr'^{field}: I[a-f0-9]+$' 529 check_re = re.compile(regex) 530 531 if options.args(): 532 raise ValueError(f'commit msg {field} check takes no options') 533 534 found = [] 535 for line in desc.splitlines(): 536 if check_re.match(line): 537 found.append(line) 538 539 if not found: 540 error = ( 541 f'Commit message is missing a "{field}:" line. It must match the\n' 542 f'following case-sensitive regex:\n\n {regex}' 543 ) 544 elif len(found) > 1: 545 error = (f'Commit message has too many "{field}:" lines. There can be ' 546 'only one.') 547 else: 548 return None 549 550 return [rh.results.HookResult(f'commit msg: "{field}:" check', 551 project, commit, error=error)] 552 553 554PREBUILT_APK_MSG = """Commit message is missing required prebuilt APK 555information. To generate the information, use the aapt tool to dump badging 556information of the APKs being uploaded, specify where the APK was built, and 557specify whether the APKs are suitable for release: 558 559 for apk in $(find . -name '*.apk' | sort); do 560 echo "${apk}" 561 ${AAPT} dump badging "${apk}" | 562 grep -iE "(package: |sdkVersion:|targetSdkVersion:)" | 563 sed -e "s/' /'\\n/g" 564 echo 565 done 566 567It must match the following case-sensitive multiline regex searches: 568 569 %s 570 571For more information, see go/platform-prebuilt and go/android-prebuilt. 572 573""" 574 575 576def check_commit_msg_prebuilt_apk_fields(project, commit, desc, diff, 577 options=None): 578 """Check that prebuilt APK commits contain the required lines.""" 579 580 if options.args(): 581 raise ValueError('prebuilt apk check takes no options') 582 583 filtered = _filter_diff(diff, [r'\.apk$']) 584 if not filtered: 585 return None 586 587 regexes = [ 588 r'^package: .*$', 589 r'^sdkVersion:.*$', 590 r'^targetSdkVersion:.*$', 591 r'^Built here:.*$', 592 (r'^This build IS( NOT)? suitable for' 593 r'( preview|( preview or)? public) release' 594 r'( but IS NOT suitable for public release)?\.$') 595 ] 596 597 missing = [] 598 for regex in regexes: 599 if not re.search(regex, desc, re.MULTILINE): 600 missing.append(regex) 601 602 if missing: 603 error = PREBUILT_APK_MSG % '\n '.join(missing) 604 else: 605 return None 606 607 return [rh.results.HookResult('commit msg: "prebuilt apk:" check', 608 project, commit, error=error)] 609 610 611TEST_MSG = """Commit message is missing a "Test:" line. It must match the 612following case-sensitive regex: 613 614 %s 615 616The Test: stanza is free-form and should describe how you tested your change. 617As a CL author, you'll have a consistent place to describe the testing strategy 618you use for your work. As a CL reviewer, you'll be reminded to discuss testing 619as part of your code review, and you'll more easily replicate testing when you 620patch in CLs locally. 621 622Some examples below: 623 624Test: make WITH_TIDY=1 mmma art 625Test: make test-art 626Test: manual - took a photo 627Test: refactoring CL. Existing unit tests still pass. 628 629Check the git history for more examples. It's a free-form field, so we urge 630you to develop conventions that make sense for your project. Note that many 631projects use exact test commands, which are perfectly fine. 632 633Adding good automated tests with new code is critical to our goals of keeping 634the system stable and constantly improving quality. Please use Test: to 635highlight this area of your development. And reviewers, please insist on 636high-quality Test: descriptions. 637""" 638 639 640def check_commit_msg_test_field(project, commit, desc, _diff, options=None): 641 """Check the commit message for a 'Test:' line.""" 642 field = 'Test' 643 regex = fr'^{field}: .*$' 644 check_re = re.compile(regex) 645 646 if options.args(): 647 raise ValueError(f'commit msg {field} check takes no options') 648 649 found = [] 650 for line in desc.splitlines(): 651 if check_re.match(line): 652 found.append(line) 653 654 if not found: 655 error = TEST_MSG % (regex) 656 else: 657 return None 658 659 return [rh.results.HookResult(f'commit msg: "{field}:" check', 660 project, commit, error=error)] 661 662 663RELNOTE_MISSPELL_MSG = """Commit message contains something that looks 664similar to the "Relnote:" tag. It must match the regex: 665 666 %s 667 668The Relnote: stanza is free-form and should describe what developers need to 669know about your change. 670 671Some examples below: 672 673Relnote: "Added a new API `Class#isBetter` to determine whether or not the 674class is better" 675Relnote: Fixed an issue where the UI would hang on a double tap. 676 677Check the git history for more examples. It's a free-form field, so we urge 678you to develop conventions that make sense for your project. 679""" 680 681RELNOTE_MISSING_QUOTES_MSG = """Commit message contains something that looks 682similar to the "Relnote:" tag but might be malformatted. For multiline 683release notes, you need to include a starting and closing quote. 684 685Multi-line Relnote example: 686 687Relnote: "Added a new API `Class#getSize` to get the size of the class. 688 This is useful if you need to know the size of the class." 689 690Single-line Relnote example: 691 692Relnote: Added a new API `Class#containsData` 693""" 694 695RELNOTE_INVALID_QUOTES_MSG = """Commit message contains something that looks 696similar to the "Relnote:" tag but might be malformatted. If you are using 697quotes that do not mark the start or end of a Relnote, you need to escape them 698with a backslash. 699 700Non-starting/non-ending quote Relnote examples: 701 702Relnote: "Fixed an error with `Class#getBar()` where \"foo\" would be returned 703in edge cases." 704Relnote: Added a new API to handle strings like \"foo\" 705""" 706 707def check_commit_msg_relnote_field_format(project, commit, desc, _diff, 708 options=None): 709 """Check the commit for one correctly formatted 'Relnote:' line. 710 711 Checks the commit message for two things: 712 (1) Checks for possible misspellings of the 'Relnote:' tag. 713 (2) Ensures that multiline release notes are properly formatted with a 714 starting quote and an endling quote. 715 (3) Checks that release notes that contain non-starting or non-ending 716 quotes are escaped with a backslash. 717 """ 718 field = 'Relnote' 719 regex_relnote = fr'^{field}:.*$' 720 check_re_relnote = re.compile(regex_relnote, re.IGNORECASE) 721 722 if options.args(): 723 raise ValueError(f'commit msg {field} check takes no options') 724 725 # Check 1: Check for possible misspellings of the `Relnote:` field. 726 727 # Regex for misspelled fields. 728 possible_field_misspells = { 729 'Relnotes', 'ReleaseNote', 730 'Rel-note', 'Rel note', 731 'rel-notes', 'releasenotes', 732 'release-note', 'release-notes', 733 } 734 re_possible_field_misspells = '|'.join(possible_field_misspells) 735 regex_field_misspells = fr'^({re_possible_field_misspells}): .*$' 736 check_re_field_misspells = re.compile(regex_field_misspells, re.IGNORECASE) 737 738 ret = [] 739 for line in desc.splitlines(): 740 if check_re_field_misspells.match(line): 741 error = RELNOTE_MISSPELL_MSG % (regex_relnote, ) 742 ret.append( 743 rh.results.HookResult( 744 f'commit msg: "{field}:" tag spelling error', 745 project, commit, error=error)) 746 747 # Check 2: Check that multiline Relnotes are quoted. 748 749 check_re_empty_string = re.compile(r'^$') 750 751 # Regex to find other fields that could be used. 752 regex_other_fields = r'^[a-zA-Z0-9-]+:' 753 check_re_other_fields = re.compile(regex_other_fields) 754 755 desc_lines = desc.splitlines() 756 for i, cur_line in enumerate(desc_lines): 757 # Look for a Relnote tag that is before the last line and 758 # lacking any quotes. 759 if (check_re_relnote.match(cur_line) and 760 i < len(desc_lines) - 1 and 761 '"' not in cur_line): 762 next_line = desc_lines[i + 1] 763 # Check that the next line does not contain any other field 764 # and it's not an empty string. 765 if (not check_re_other_fields.findall(next_line) and 766 not check_re_empty_string.match(next_line)): 767 ret.append( 768 rh.results.HookResult( 769 f'commit msg: "{field}:" tag missing quotes', 770 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 771 break 772 773 # Check 3: Check that multiline Relnotes contain matching quotes. 774 first_quote_found = False 775 second_quote_found = False 776 for cur_line in desc_lines: 777 contains_quote = '"' in cur_line 778 contains_field = check_re_other_fields.findall(cur_line) 779 # If we have found the first quote and another field, break and fail. 780 if first_quote_found and contains_field: 781 break 782 # If we have found the first quote, this line contains a quote, 783 # and this line is not another field, break and succeed. 784 if first_quote_found and contains_quote: 785 second_quote_found = True 786 break 787 # Check that the `Relnote:` tag exists and it contains a starting quote. 788 if check_re_relnote.match(cur_line) and contains_quote: 789 first_quote_found = True 790 # A single-line Relnote containing a start and ending triple quote 791 # is valid. 792 if cur_line.count('"""') == 2: 793 second_quote_found = True 794 break 795 # A single-line Relnote containing a start and ending quote 796 # is valid. 797 if cur_line.count('"') - cur_line.count('\\"') == 2: 798 second_quote_found = True 799 break 800 if first_quote_found != second_quote_found: 801 ret.append( 802 rh.results.HookResult( 803 f'commit msg: "{field}:" tag missing closing quote', 804 project, commit, error=RELNOTE_MISSING_QUOTES_MSG)) 805 806 # Check 4: Check that non-starting or non-ending quotes are escaped with a 807 # backslash. 808 line_needs_checking = False 809 uses_invalid_quotes = False 810 for cur_line in desc_lines: 811 if check_re_other_fields.findall(cur_line): 812 line_needs_checking = False 813 on_relnote_line = check_re_relnote.match(cur_line) 814 # Determine if we are parsing the base `Relnote:` line. 815 if on_relnote_line and '"' in cur_line: 816 line_needs_checking = True 817 # We don't think anyone will type '"""' and then forget to 818 # escape it, so we're not checking for this. 819 if '"""' in cur_line: 820 break 821 if line_needs_checking: 822 stripped_line = re.sub(fr'^{field}:', '', cur_line, 823 flags=re.IGNORECASE).strip() 824 for i, character in enumerate(stripped_line): 825 if i == 0: 826 # Case 1: Valid quote at the beginning of the 827 # base `Relnote:` line. 828 if on_relnote_line: 829 continue 830 # Case 2: Invalid quote at the beginning of following 831 # lines, where we are not terminating the release note. 832 if character == '"' and stripped_line != '"': 833 uses_invalid_quotes = True 834 break 835 # Case 3: Check all other cases. 836 if (character == '"' 837 and 0 < i < len(stripped_line) - 1 838 and stripped_line[i-1] != '"' 839 and stripped_line[i-1] != "\\"): 840 uses_invalid_quotes = True 841 break 842 843 if uses_invalid_quotes: 844 ret.append(rh.results.HookResult( 845 f'commit msg: "{field}:" tag using unescaped quotes', 846 project, commit, error=RELNOTE_INVALID_QUOTES_MSG)) 847 return ret 848 849 850RELNOTE_REQUIRED_CURRENT_TXT_MSG = """\ 851Commit contains a change to current.txt or public_plus_experimental_current.txt, 852but the commit message does not contain the required `Relnote:` tag. It must 853match the regex: 854 855 %s 856 857The Relnote: stanza is free-form and should describe what developers need to 858know about your change. If you are making infrastructure changes, you 859can set the Relnote: stanza to be "N/A" for the commit to not be included 860in release notes. 861 862Some examples: 863 864Relnote: "Added a new API `Class#isBetter` to determine whether or not the 865class is better" 866Relnote: Fixed an issue where the UI would hang on a double tap. 867Relnote: N/A 868 869Check the git history for more examples. 870""" 871 872def check_commit_msg_relnote_for_current_txt(project, commit, desc, diff, 873 options=None): 874 """Check changes to current.txt contain the 'Relnote:' stanza.""" 875 field = 'Relnote' 876 regex = fr'^{field}: .+$' 877 check_re = re.compile(regex, re.IGNORECASE) 878 879 if options.args(): 880 raise ValueError(f'commit msg {field} check takes no options') 881 882 filtered = _filter_diff( 883 diff, 884 [r'(^|/)(public_plus_experimental_current|current)\.txt$'] 885 ) 886 # If the commit does not contain a change to *current.txt, then this repo 887 # hook check no longer applies. 888 if not filtered: 889 return None 890 891 found = [] 892 for line in desc.splitlines(): 893 if check_re.match(line): 894 found.append(line) 895 896 if not found: 897 error = RELNOTE_REQUIRED_CURRENT_TXT_MSG % (regex) 898 else: 899 return None 900 901 return [rh.results.HookResult(f'commit msg: "{field}:" check', 902 project, commit, error=error)] 903 904 905def check_cpplint(project, commit, _desc, diff, options=None): 906 """Run cpplint.""" 907 # This list matches what cpplint expects. We could run on more (like .cxx), 908 # but cpplint would just ignore them. 909 filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$']) 910 if not filtered: 911 return None 912 913 cpplint = options.tool_path('cpplint') 914 cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered) 915 return _check_cmd('cpplint', project, commit, cmd) 916 917 918def check_gofmt(project, commit, _desc, diff, options=None): 919 """Checks that Go files are formatted with gofmt.""" 920 filtered = _filter_diff(diff, [r'\.go$']) 921 if not filtered: 922 return None 923 924 gofmt = options.tool_path('gofmt') 925 cmd = [gofmt, '-l'] + options.args() 926 fixup_cmd = [gofmt, '-w'] + options.args() 927 928 ret = [] 929 for d in filtered: 930 data = rh.git.get_file_content(commit, d.file) 931 result = _run(cmd, input=data) 932 if result.stdout: 933 ret.append(rh.results.HookResult( 934 'gofmt', project, commit, error=result.stdout, 935 files=(d.file,), fixup_cmd=fixup_cmd)) 936 return ret 937 938 939def check_json(project, commit, _desc, diff, options=None): 940 """Verify json files are valid.""" 941 if options.args(): 942 raise ValueError('json check takes no options') 943 944 filtered = _filter_diff(diff, [r'\.json$']) 945 if not filtered: 946 return None 947 948 ret = [] 949 for d in filtered: 950 data = rh.git.get_file_content(commit, d.file) 951 try: 952 json.loads(data) 953 except ValueError as e: 954 ret.append(rh.results.HookResult( 955 'json', project, commit, error=str(e), 956 files=(d.file,))) 957 return ret 958 959 960def _check_pylint(project, commit, _desc, diff, extra_args=None, options=None): 961 """Run pylint.""" 962 filtered = _filter_diff(diff, [r'\.py$']) 963 if not filtered: 964 return None 965 966 if extra_args is None: 967 extra_args = [] 968 969 pylint = options.tool_path('pylint') 970 cmd = [ 971 get_helper_path('pylint.py'), 972 '--executable-path', pylint, 973 ] + extra_args + options.args(('${PREUPLOAD_FILES}',), filtered) 974 return _check_cmd('pylint', project, commit, cmd) 975 976 977def check_pylint2(project, commit, desc, diff, options=None): 978 """Run pylint through Python 2.""" 979 return _check_pylint(project, commit, desc, diff, options=options) 980 981 982def check_pylint3(project, commit, desc, diff, options=None): 983 """Run pylint through Python 3.""" 984 return _check_pylint(project, commit, desc, diff, 985 extra_args=['--py3'], 986 options=options) 987 988 989def check_rustfmt(project, commit, _desc, diff, options=None): 990 """Run "rustfmt --check" on diffed rust files""" 991 filtered = _filter_diff(diff, [r'\.rs$']) 992 if not filtered: 993 return None 994 995 rustfmt = options.tool_path('rustfmt') 996 cmd = [rustfmt] + options.args((), filtered) 997 ret = [] 998 for d in filtered: 999 data = rh.git.get_file_content(commit, d.file) 1000 result = _run(cmd, input=data) 1001 # If the parsing failed, stdout will contain enough details on the 1002 # location of the error. 1003 if result.returncode: 1004 ret.append(rh.results.HookResult( 1005 'rustfmt', project, commit, error=result.stdout, 1006 files=(d.file,))) 1007 continue 1008 # TODO(b/164111102): rustfmt stable does not support --check on stdin. 1009 # If no error is reported, compare stdin with stdout. 1010 if data != result.stdout: 1011 ret.append(rh.results.HookResult( 1012 'rustfmt', project, commit, error='Files not formatted', 1013 files=(d.file,), fixup_cmd=cmd)) 1014 return ret 1015 1016 1017def check_xmllint(project, commit, _desc, diff, options=None): 1018 """Run xmllint.""" 1019 # XXX: Should we drop most of these and probe for <?xml> tags? 1020 extensions = frozenset(( 1021 'dbus-xml', # Generated DBUS interface. 1022 'dia', # File format for Dia. 1023 'dtd', # Document Type Definition. 1024 'fml', # Fuzzy markup language. 1025 'form', # Forms created by IntelliJ GUI Designer. 1026 'fxml', # JavaFX user interfaces. 1027 'glade', # Glade user interface design. 1028 'grd', # GRIT translation files. 1029 'iml', # Android build modules? 1030 'kml', # Keyhole Markup Language. 1031 'mxml', # Macromedia user interface markup language. 1032 'nib', # OS X Cocoa Interface Builder. 1033 'plist', # Property list (for OS X). 1034 'pom', # Project Object Model (for Apache Maven). 1035 'rng', # RELAX NG schemas. 1036 'sgml', # Standard Generalized Markup Language. 1037 'svg', # Scalable Vector Graphics. 1038 'uml', # Unified Modeling Language. 1039 'vcproj', # Microsoft Visual Studio project. 1040 'vcxproj', # Microsoft Visual Studio project. 1041 'wxs', # WiX Transform File. 1042 'xhtml', # XML HTML. 1043 'xib', # OS X Cocoa Interface Builder. 1044 'xlb', # Android locale bundle. 1045 'xml', # Extensible Markup Language. 1046 'xsd', # XML Schema Definition. 1047 'xsl', # Extensible Stylesheet Language. 1048 )) 1049 1050 filtered = _filter_diff(diff, [r'\.(' + '|'.join(extensions) + r')$']) 1051 if not filtered: 1052 return None 1053 1054 # TODO: Figure out how to integrate schema validation. 1055 # XXX: Should we use python's XML libs instead? 1056 cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered) 1057 1058 return _check_cmd('xmllint', project, commit, cmd) 1059 1060 1061def check_android_test_mapping(project, commit, _desc, diff, options=None): 1062 """Verify Android TEST_MAPPING files are valid.""" 1063 if options.args(): 1064 raise ValueError('Android TEST_MAPPING check takes no options') 1065 filtered = _filter_diff(diff, [r'(^|.*/)TEST_MAPPING$']) 1066 if not filtered: 1067 return None 1068 1069 testmapping_format = options.tool_path('android-test-mapping-format') 1070 testmapping_args = ['--commit', commit] 1071 cmd = [testmapping_format] + options.args( 1072 (project.dir, '${PREUPLOAD_FILES}'), filtered) + testmapping_args 1073 return _check_cmd('android-test-mapping-format', project, commit, cmd) 1074 1075 1076def check_aidl_format(project, commit, _desc, diff, options=None): 1077 """Checks that AIDL files are formatted with aidl-format.""" 1078 # All *.aidl files except for those under aidl_api directory. 1079 filtered = _filter_diff(diff, [r'\.aidl$'], [r'(^|/)aidl_api/']) 1080 if not filtered: 1081 return None 1082 aidl_format = options.tool_path('aidl-format') 1083 clang_format = options.tool_path('clang-format') 1084 diff_cmd = [aidl_format, '-d', '--clang-format-path', clang_format] + \ 1085 options.args((), filtered) 1086 ret = [] 1087 for d in filtered: 1088 data = rh.git.get_file_content(commit, d.file) 1089 result = _run(diff_cmd, input=data) 1090 if result.stdout: 1091 fixup_cmd = [aidl_format, '-w', '--clang-format-path', clang_format] 1092 ret.append(rh.results.HookResult( 1093 'aidl-format', project, commit, error=result.stdout, 1094 files=(d.file,), fixup_cmd=fixup_cmd)) 1095 return ret 1096 1097 1098# Hooks that projects can opt into. 1099# Note: Make sure to keep the top level README.md up to date when adding more! 1100BUILTIN_HOOKS = { 1101 'aidl_format': check_aidl_format, 1102 'android_test_mapping_format': check_android_test_mapping, 1103 'aosp_license': check_aosp_license, 1104 'bpfmt': check_bpfmt, 1105 'checkpatch': check_checkpatch, 1106 'clang_format': check_clang_format, 1107 'commit_msg_bug_field': check_commit_msg_bug_field, 1108 'commit_msg_changeid_field': check_commit_msg_changeid_field, 1109 'commit_msg_prebuilt_apk_fields': check_commit_msg_prebuilt_apk_fields, 1110 'commit_msg_relnote_field_format': check_commit_msg_relnote_field_format, 1111 'commit_msg_relnote_for_current_txt': 1112 check_commit_msg_relnote_for_current_txt, 1113 'commit_msg_test_field': check_commit_msg_test_field, 1114 'cpplint': check_cpplint, 1115 'gofmt': check_gofmt, 1116 'google_java_format': check_google_java_format, 1117 'jsonlint': check_json, 1118 'ktfmt': check_ktfmt, 1119 'pylint': check_pylint3, 1120 'pylint2': check_pylint2, 1121 'pylint3': check_pylint3, 1122 'rustfmt': check_rustfmt, 1123 'xmllint': check_xmllint, 1124} 1125 1126# Additional tools that the hooks can call with their default values. 1127# Note: Make sure to keep the top level README.md up to date when adding more! 1128TOOL_PATHS = { 1129 'aidl-format': 'aidl-format', 1130 'android-test-mapping-format': 1131 os.path.join(TOOLS_DIR, 'android_test_mapping_format.py'), 1132 'bpfmt': 'bpfmt', 1133 'clang-format': 'clang-format', 1134 'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'), 1135 'git-clang-format': 'git-clang-format', 1136 'gofmt': 'gofmt', 1137 'google-java-format': 'google-java-format', 1138 'google-java-format-diff': 'google-java-format-diff.py', 1139 'ktfmt': 'ktfmt', 1140 'pylint': 'pylint', 1141 'rustfmt': 'rustfmt', 1142} 1143