1#!/usr/bin/env python3 2# Copyright 2022 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# Contains custom presubmit checks implemented in python. 7# 8# These are implemented as a separate CLI tool from tools/presubmit as the presubmit 9# framework needs to call a subprocess to execute checks. 10 11from fnmatch import fnmatch 12import os 13import re 14import json 15from datetime import datetime 16from pathlib import Path 17from typing import Dict, Generator, List, cast 18 19from impl.common import ( 20 cmd, 21 cwd_context, 22 run_commands, 23) 24 25 26def check_platform_independent(*files: str): 27 "Checks the provided files to ensure they are free of platform independent code." 28 cfg_unix = "cfg.*unix" 29 cfg_linux = "cfg.*linux" 30 cfg_windows = "cfg.*windows" 31 cfg_android = "cfg.*android" 32 target_os = "target_os = " 33 34 target_os_pattern = re.compile( 35 "%s|%s|%s|%s|%s" % (cfg_android, cfg_linux, cfg_unix, cfg_windows, target_os) 36 ) 37 38 for file in files: 39 for line_number, line in enumerate(open(file, encoding="utf8")): 40 if re.search(target_os_pattern, line): 41 raise Exception(f"Found unexpected platform dependent code in {file}:{line_number}") 42 43 44CRLF_LINE_ENDING_FILES: List[str] = [ 45 "**.bat", 46 "**.ps1", 47 "e2e_tests/tests/goldens/backcompat_test_simple_lspci_win.txt", 48 "tools/windows/build_test", 49] 50 51 52def is_crlf_file(file: str): 53 for glob in CRLF_LINE_ENDING_FILES: 54 if fnmatch(file, glob): 55 return True 56 return False 57 58 59def check_line_endings(*files: str): 60 "Checks line endings. Windows only files are using clrf. All others just lf." 61 for line in cmd("git ls-files --eol", *files).lines(): 62 parts = line.split() 63 file = parts[-1] 64 index_endings = parts[0][2:] 65 wdir_endings = parts[1][2:] 66 67 def check_endings(endings: str): 68 if is_crlf_file(file): 69 if endings not in ("crlf", "mixed"): 70 raise Exception(f"{file} Expected crlf file endings. Found {endings}") 71 else: 72 if endings in ("crlf", "mixed"): 73 raise Exception(f"{file} Expected lf file endings. Found {endings}") 74 75 check_endings(index_endings) 76 check_endings(wdir_endings) 77 78 79def check_rust_lockfiles(*files: str): 80 "Verifies that none of the Cargo.lock files require updates." 81 lockfiles = [Path("Cargo.lock"), *Path("common").glob("*/Cargo.lock")] 82 for path in lockfiles: 83 with cwd_context(path.parent): 84 if not cmd("cargo update --workspace --locked").success(): 85 print(f"{path} is not up-to-date.") 86 print() 87 print("You may need to rebase your changes and run `cargo update --workspace`") 88 print("(or ./tools/run_tests) to ensure the Cargo.lock file is current.") 89 raise Exception("Cargo.lock out of date") 90 91 92# These crosvm features are currently not built upstream. Do not add to this list. 93KNOWN_DISABLED_FEATURES = [ 94 "default-no-sandbox", 95 "gvm", 96 "libvda", 97 "perfetto", 98 "process-invariants", 99 "prod-build", 100 "sandbox", 101 "seccomp_trace", 102 "slirp-ring-capture", 103 "vulkano", 104 "whpx", 105] 106 107 108def check_rust_features(*files: str): 109 "Verifies that all cargo features are included in the list of features we compile upstream." 110 metadata = json.loads(cmd("cargo metadata --format-version=1").stdout()) 111 crosvm_metadata = next(p for p in metadata["packages"] if p["name"] == "crosvm") 112 features = cast(Dict[str, List[str]], crosvm_metadata["features"]) 113 114 def collect_features(feature_name: str) -> Generator[str, None, None]: 115 yield feature_name 116 for feature in features[feature_name]: 117 if feature in features: 118 yield from collect_features(feature) 119 else: 120 # optional crate is enabled through sub-feature of the crate. 121 # e.g. protos optional crate/feature is enabled by protos/plugin 122 optional_crate_name = feature.split("/")[0] 123 if ( 124 optional_crate_name in features 125 and features[optional_crate_name][0] == f"dep:{optional_crate_name}" 126 ): 127 yield optional_crate_name 128 129 all_platform_features = set( 130 ( 131 *collect_features("all-x86_64"), 132 *collect_features("all-aarch64"), 133 *collect_features("all-armhf"), 134 *collect_features("all-mingw64"), 135 *collect_features("all-msvc64"), 136 *collect_features("all-riscv64"), 137 *collect_features("all-android"), 138 ) 139 ) 140 disabled_features = [ 141 feature 142 for feature in features 143 if feature not in all_platform_features and feature not in KNOWN_DISABLED_FEATURES 144 ] 145 if disabled_features: 146 raise Exception( 147 f"The features {', '.join(disabled_features)} are not enabled in upstream crosvm builds." 148 ) 149 150 151LICENSE_HEADER_RE = ( 152 r".*Copyright (?P<year>20[0-9]{2})(?:-20[0-9]{2})? The ChromiumOS Authors\n" 153 r".*Use of this source code is governed by a BSD-style license that can be\n" 154 r".*found in the LICENSE file\.\n" 155 r"( *\*/\n)?" # allow the end of a C-style comment before the blank line 156 r"\n" 157) 158 159NEW_LICENSE_HEADER = [ 160 f"Copyright {datetime.now().year} The ChromiumOS Authors", 161 "Use of this source code is governed by a BSD-style license that can be", 162 "found in the LICENSE file.", 163] 164 165 166def new_licence_header(file_suffix: str): 167 if file_suffix in (".py", "", ".policy", ".sh"): 168 prefix = "#" 169 else: 170 prefix = "//" 171 return "\n".join(f"{prefix} {line}" for line in NEW_LICENSE_HEADER) + "\n\n" 172 173 174def check_copyright_header(*files: str, fix: bool = False): 175 "Checks copyright header. Can 'fix' them if needed by adding the header." 176 license_re = re.compile(LICENSE_HEADER_RE, re.MULTILINE) 177 for file_path in (Path(f) for f in files): 178 header = file_path.open("r").read(512) 179 license_match = license_re.search(header) 180 if license_match: 181 continue 182 # Generated files do not need a copyright header. 183 if "generated by" in header: 184 continue 185 if fix: 186 print(f"Adding copyright header: {file_path}") 187 contents = file_path.read_text() 188 file_path.write_text(new_licence_header(file_path.suffix) + contents) 189 else: 190 raise Exception(f"Bad copyright header: {file_path}") 191 192 193def check_file_ends_with_newline(*files: str, fix: bool = False): 194 "Checks if files end with a newline." 195 for file_path in (Path(f) for f in files): 196 with file_path.open("rb") as file: 197 # Skip empty files 198 file.seek(0, os.SEEK_END) 199 if file.tell() == 0: 200 continue 201 # Check last byte of the file 202 file.seek(-1, os.SEEK_END) 203 file_end = file.read(1) 204 if file_end.decode("utf-8") != "\n": 205 if fix: 206 file_path.write_text(file_path.read_text() + "\n") 207 else: 208 raise Exception(f"File does not end with a newline {file_path}") 209 210 211if __name__ == "__main__": 212 run_commands( 213 check_file_ends_with_newline, 214 check_copyright_header, 215 check_rust_features, 216 check_rust_lockfiles, 217 check_line_endings, 218 check_platform_independent, 219 ) 220