1#!/usr/bin/env python3 2# Copyright 2023 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import copy 7import os 8from pathlib import Path 9import sys 10from typing import Any, Iterable, List, Optional, Union 11from impl.common import ( 12 CROSVM_ROOT, 13 TOOLS_ROOT, 14 Command, 15 Remote, 16 quoted, 17 Styles, 18 argh, 19 console, 20 chdir, 21 cmd, 22 record_time, 23 run_main, 24 sudo_is_passwordless, 25 verbose, 26 Triple, 27) 28from impl.test_config import ROOT_TESTS, DO_NOT_RUN, DO_NOT_RUN_AARCH64, DO_NOT_RUN_WIN64, E2E_TESTS 29from impl.test_config import DO_NOT_BUILD_RISCV64, DO_NOT_RUN_WINE64 30from impl import testvm 31 32rsync = cmd("rsync") 33cargo = cmd("cargo") 34 35# Name of the directory used to package all test files. 36PACKAGE_NAME = "integration_tests_package" 37 38 39def join_filters(items: Iterable[str], op: str): 40 return op.join(f"({i})" for i in items) 41 42 43class TestFilter(object): 44 """ 45 Utility structure to join user-provided filter expressions with additional filters 46 47 See https://nexte.st/book/filter-expressions.html 48 """ 49 50 def __init__(self, expression: str): 51 self.expression = expression 52 53 def exclude(self, *exclude_exprs: str): 54 return self.subset(f"not ({join_filters(exclude_exprs, '|')})") 55 56 def include(self, *include_exprs: str): 57 include_expr = join_filters(include_exprs, "|") 58 return TestFilter(f"({self.expression}) | ({include_expr})") 59 60 def subset(self, *subset_exprs: str): 61 subset_expr = join_filters(subset_exprs, "|") 62 if not self.expression: 63 return TestFilter(subset_expr) 64 return TestFilter(f"({self.expression}) & ({subset_expr})") 65 66 def to_args(self): 67 if not self.expression: 68 return 69 yield "--filter-expr" 70 yield quoted(self.expression) 71 72 73def configure_cargo( 74 cmd: Command, triple: Triple, features: Optional[str], no_default_features: bool 75): 76 "Configures the provided cmd with cargo arguments and environment needed to build for triple." 77 return ( 78 cmd.with_args( 79 "--workspace", 80 "--no-default-features" if no_default_features else None, 81 f"--features={features}" if features else None, 82 ) 83 .with_color_flag() 84 .with_envs(triple.get_cargo_env()) 85 ) 86 87 88class HostTarget(object): 89 def __init__(self, package_dir: Path): 90 self.run_cmd = cmd(package_dir / "run.sh").with_color_flag() 91 92 def run_tests(self, extra_args: List[Any]): 93 return self.run_cmd.with_args(*extra_args).fg(style=Styles.live_truncated(), check=False) 94 95 96class SshTarget(object): 97 def __init__(self, package_archive: Path, remote: Remote): 98 console.print("Transfering integration tests package...") 99 with record_time("Transfering"): 100 remote.scp([package_archive], "") 101 with record_time("Unpacking"): 102 remote.ssh(cmd("tar xaf", package_archive.name)).fg(style=Styles.live_truncated()) 103 self.remote_run_cmd = cmd(f"{PACKAGE_NAME}/run.sh").with_color_flag() 104 self.remote = remote 105 106 def run_tests(self, extra_args: List[Any]): 107 return self.remote.ssh(self.remote_run_cmd.with_args(*extra_args)).fg( 108 style=Styles.live_truncated(), 109 check=False, 110 ) 111 112 113def check_host_prerequisites(run_root_tests: bool): 114 "Check various prerequisites for executing test binaries." 115 if os.name == "nt": 116 return 117 118 if run_root_tests: 119 console.print("Running tests that require root privileges. Refreshing sudo now.") 120 cmd("sudo true").fg() 121 122 for device in ["/dev/kvm", "/dev/vhost-vsock"]: 123 if not os.access(device, os.R_OK | os.W_OK): 124 console.print(f"{device} access is required", style="red") 125 sys.exit(1) 126 127 128def check_build_prerequisites(triple: Triple): 129 installed_toolchains = cmd("rustup target list --installed").lines() 130 if str(triple) not in installed_toolchains: 131 console.print(f"Your host is not configured to build for [green]{triple}[/green]") 132 console.print(f"[green]Tip:[/green] Run tests in the dev container with:") 133 console.print() 134 console.print( 135 f" [blue]$ tools/dev_container tools/run_tests {' '.join(sys.argv[1:])}[/blue]" 136 ) 137 sys.exit(1) 138 139 140def get_vm_arch(triple: Triple): 141 if str(triple) == "x86_64-unknown-linux-gnu": 142 return "x86_64" 143 elif str(triple) == "aarch64-unknown-linux-gnu": 144 return "aarch64" 145 elif str(triple) == "riscv64gc-unknown-linux-gnu": 146 return "riscv64" 147 else: 148 raise Exception(f"{triple} is not supported for running tests in a VM.") 149 150 151@argh.arg("--filter-expr", "-E", type=str, action="append", help="Nextest filter expression.") 152@argh.arg( 153 "--platform", "-p", help="Which platform to test. (x86_64, aarch64, armhw, mingw64, riscv64)" 154) 155@argh.arg("--dut", help="Which device to test on. (vm or host)") 156@argh.arg("--no-default-features", help="Don't enable default features") 157@argh.arg("--no-run", "--build-only", help="Build only, do not run any tests.") 158@argh.arg("--no-unit-tests", help="Do not run unit tests.") 159@argh.arg("--no-integration-tests", help="Do not run integration tests.") 160@argh.arg("--no-strip", help="Do not strip test binaries of debug info.") 161@argh.arg("--run-root-tests", help="Enables integration tests that require root privileges.") 162@argh.arg( 163 "--features", 164 help=f"List of comma separated features to be passed to cargo. Defaults to `all-$platform`", 165) 166@argh.arg("--no-parallel", help="Do not parallelize integration tests. Slower but more stable.") 167@argh.arg("--repetitions", help="Repeat all tests, useful for checking test stability.") 168def main( 169 filter_expr: List[str] = [], 170 platform: Optional[str] = None, 171 dut: Optional[str] = None, 172 no_default_features: bool = False, 173 no_run: bool = False, 174 no_unit_tests: bool = False, 175 no_integration_tests: bool = False, 176 no_strip: bool = False, 177 run_root_tests: bool = False, 178 features: Optional[str] = None, 179 no_parallel: bool = False, 180 repetitions: int = 1, 181): 182 """ 183 Runs all crosvm tests 184 185 For details on how crosvm tests are organized, see https://crosvm.dev/book/testing/index.html 186 187 # Basic Usage 188 189 To run all unit tests for the hosts native architecture: 190 191 $ ./tools/run_tests 192 193 To run all unit tests for another supported architecture using an emulator (e.g. wine64, 194 qemu user space emulation). 195 196 $ ./tools/run_tests -p aarch64 197 $ ./tools/run_tests -p armhw 198 $ ./tools/run_tests -p mingw64 199 200 # Integration Tests 201 202 Integration tests can be run on a built-in virtual machine: 203 204 $ ./tools/run_tests --dut=vm 205 $ ./tools/run_tests --dut=vm -p aarch64 206 207 The virtual machine is automatically started for the test process and can be managed via the 208 `./tools/x86vm` or `./tools/aarch64vm` tools. 209 210 Integration tests can be run on the host machine as well, but cannot be guaranteed to work on 211 all configurations. 212 213 $ ./tools/run_tests --dut=host 214 215 # Test Filtering 216 217 This script supports nextest filter expressions: https://nexte.st/book/filter-expressions.html 218 219 For example to run all tests in `my-crate` and all crates that depend on it: 220 221 $ ./tools/run_tests [--dut=] -E 'rdeps(my-crate)' 222 """ 223 chdir(CROSVM_ROOT) 224 225 if os.name == "posix" and not cmd("which cargo-nextest").success(): 226 raise Exception("Cannot find cargo-nextest. Please re-run `./tools/install-deps`") 227 elif os.name == "nt" and not cmd("where.exe cargo-nextest.exe").success(): 228 raise Exception("Cannot find cargo-nextest. Please re-run `./tools/install-deps.ps1`") 229 230 triple = Triple.from_shorthand(platform) if platform else Triple.host_default() 231 232 test_filter = TestFilter(join_filters(filter_expr, "|")) 233 234 if not features and not no_default_features: 235 features = triple.feature_flag 236 237 if no_run: 238 no_integration_tests = True 239 no_unit_tests = True 240 241 # Disable the DUT if integration tests are not run. 242 if no_integration_tests: 243 dut = None 244 245 # Automatically enable tests that require root if sudo is passwordless 246 if not run_root_tests: 247 if dut == "host": 248 run_root_tests = sudo_is_passwordless() 249 elif dut == "vm": 250 # The test VMs have passwordless sudo configured. 251 run_root_tests = True 252 253 # Print summary of tests and where they will be executed. 254 if dut == "host": 255 dut_str = "Run on host" 256 elif dut == "vm" and os.name == "posix": 257 dut_str = f"Run on built-in {get_vm_arch(triple)} vm" 258 elif dut == None: 259 dut_str = "[yellow]Skip[/yellow]" 260 else: 261 raise Exception( 262 f"--dut={dut} is not supported. Options are --dut=host or --dut=vm (linux only)" 263 ) 264 265 skip_str = "[yellow]skip[/yellow]" 266 unit_test_str = "Run on host" if not no_unit_tests else skip_str 267 integration_test_str = dut_str if dut else skip_str 268 profile = os.environ.get("NEXTEST_PROFILE", "default") 269 console.print(f"Running tests for [green]{triple}[/green]") 270 console.print(f"Profile: [green]{profile}[/green]") 271 console.print(f"With features: [green]{features}[/green]") 272 console.print(f"no-default-features: [green]{no_default_features}[/green]") 273 console.print() 274 console.print(f" Unit tests: [bold]{unit_test_str}[/bold]") 275 console.print(f" Integration tests: [bold]{integration_test_str}[/bold]") 276 console.print() 277 278 check_build_prerequisites(triple) 279 280 # Print tips in certain configurations. 281 if dut and not run_root_tests: 282 console.print( 283 "[green]Tip:[/green] Skipping tests that require root privileges. " 284 + "Use [bold]--run-root-tests[/bold] to enable them." 285 ) 286 if not dut: 287 console.print( 288 "[green]Tip:[/green] To run integration tests on a built-in VM: " 289 + "Use [bold]--dut=vm[/bold] (preferred)" 290 ) 291 console.print( 292 "[green]Tip:[/green] To run integration tests on the host: Use " 293 + "[bold]--dut=host[/bold] (fast, but unreliable)" 294 ) 295 if dut == "vm": 296 vm_arch = get_vm_arch(triple) 297 if vm_arch == "x86_64": 298 cli_tool = "tools/x86vm" 299 elif vm_arch == "aarch64": 300 cli_tool = "tools/aarch64vm" 301 else: 302 raise Exception(f"Unknown vm arch '{vm_arch}'") 303 console.print( 304 f"[green]Tip:[/green] The test VM will remain alive between tests. You can manage this VM with [bold]{cli_tool}[/bold]" 305 ) 306 307 # Prepare the dut for test execution 308 if dut == "host": 309 check_host_prerequisites(run_root_tests) 310 if dut == "vm": 311 # Start VM ahead of time but don't wait for it to boot. 312 testvm.up(get_vm_arch(triple)) 313 314 nextest_args = [ 315 f"--profile={profile}" if profile else None, 316 "--verbose" if verbose() else None, 317 ] 318 319 console.print() 320 console.rule("Building tests") 321 322 if triple == Triple.from_shorthand("riscv64"): 323 nextest_args += ["--exclude=" + s for s in DO_NOT_BUILD_RISCV64] 324 325 nextest_run = configure_cargo( 326 cmd("cargo nextest run"), triple, features, no_default_features 327 ).with_args(*nextest_args) 328 329 with record_time("Build"): 330 returncode = nextest_run.with_args("--no-run").fg( 331 style=Styles.live_truncated(), check=False 332 ) 333 if returncode != 0: 334 sys.exit(returncode) 335 336 if not no_unit_tests: 337 unit_test_filter = copy.deepcopy(test_filter).exclude(*E2E_TESTS).include("kind(bench)") 338 if triple == Triple.from_shorthand("mingw64") and os.name == "posix": 339 unit_test_filter = unit_test_filter.exclude(*DO_NOT_RUN_WINE64) 340 console.print() 341 console.rule("Running unit tests") 342 with record_time("Unit Tests"): 343 for i in range(repetitions): 344 if repetitions > 1: 345 console.rule(f"Round {i}", style="grey") 346 347 returncode = nextest_run.with_args("--lib --bins", *unit_test_filter.to_args()).fg( 348 style=Styles.live_truncated(), check=False 349 ) 350 if returncode != 0: 351 sys.exit(returncode) 352 353 if dut: 354 package_dir = triple.target_dir / PACKAGE_NAME 355 package_archive = package_dir.with_suffix(".tar.zst") 356 nextest_package = configure_cargo( 357 cmd(TOOLS_ROOT / "nextest_package"), triple, features, no_default_features 358 ) 359 360 test_exclusions = [*DO_NOT_RUN] 361 if not run_root_tests: 362 test_exclusions += ROOT_TESTS 363 if triple == Triple.from_shorthand("mingw64"): 364 test_exclusions += DO_NOT_RUN_WIN64 365 if os.name == "posix": 366 test_exclusions += DO_NOT_RUN_WINE64 367 if triple == Triple.from_shorthand("aarch64"): 368 test_exclusions += DO_NOT_RUN_AARCH64 369 test_filter = test_filter.exclude(*test_exclusions) 370 371 console.print() 372 console.rule("Packaging integration tests") 373 with record_time("Packing"): 374 nextest_package( 375 "--test *", 376 f"-d {package_dir}", 377 f"-o {package_archive}" if dut != "host" else None, 378 "--no-strip" if no_strip else None, 379 *test_filter.to_args(), 380 "--verbose" if verbose() else None, 381 ).fg(style=Styles.live_truncated()) 382 383 target: Union[HostTarget, SshTarget] 384 if dut == "host": 385 target = HostTarget(package_dir) 386 elif dut == "vm": 387 testvm.up(get_vm_arch(triple), wait=True) 388 remote = Remote("localhost", testvm.ssh_opts(get_vm_arch(triple))) 389 target = SshTarget(package_archive, remote) 390 391 console.print() 392 console.rule("Running integration tests") 393 with record_time("Integration tests"): 394 for i in range(repetitions): 395 if repetitions > 1: 396 console.rule(f"Round {i}", style="grey") 397 returncode = target.run_tests( 398 [ 399 *test_filter.to_args(), 400 *nextest_args, 401 "--test-threads=1" if no_parallel else None, 402 ] 403 ) 404 if returncode != 0: 405 if not no_parallel: 406 console.print( 407 "[green]Tip:[/green] Tests may fail when run in parallel on some platforms. " 408 + "Try re-running with `--no-parallel`" 409 ) 410 if dut == "host": 411 console.print( 412 f"[yellow]Tip:[/yellow] Running tests on the host may not be reliable. " 413 "Prefer [bold]--dut=vm[/bold]." 414 ) 415 sys.exit(returncode) 416 417 418if __name__ == "__main__": 419 run_main(main) 420