1#!/usr/bin/env python3 2# Copyright 2021 The ChromiumOS Authors 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6# Usage: 7# 8# To get an interactive shell for development: 9# ./tools/dev_container 10# 11# To run a command in the container, e.g. to run presubmits: 12# ./tools/dev_container ./tools/presubmit 13# 14# The state of the container (including build artifacts) are preserved between 15# calls. To stop the container call: 16# ./tools/dev_container --stop 17# 18# The dev container can also be called with a fresh container for each call that 19# is cleaned up afterwards (e.g. when run by Kokoro): 20# 21# ./tools/dev_container --hermetic CMD 22# 23# There's an alternative container which can be used to test crosvm in crOS tree. 24# It can be launched with: 25# ./tools/dev_container --cros 26 27import argparse 28from pathlib import Path 29import shutil 30from impl.util import ( 31 add_common_args, 32 confirm, 33 cros_repo_root, 34 CROSVM_ROOT, 35 is_cros_repo, 36 is_kiwi_repo, 37 kiwi_repo_root, 38 is_aosp_repo, 39 aosp_repo_root, 40) 41from impl.command import ( 42 chdir, 43 cmd, 44 quoted, 45) 46from typing import Optional, List 47import getpass 48import sys 49import unittest 50import os 51import zlib 52 53DEV_CONTAINER_NAME = ( 54 f"crosvm_dev_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}" 55) 56CROS_CONTAINER_NAME = ( 57 f"crosvm_cros_{getpass.getuser()}_{zlib.crc32(os.path.realpath(__file__).encode('utf-8')):x}" 58) 59 60DEV_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_dev" 61CROS_IMAGE_NAME = "gcr.io/crosvm-infra/crosvm_cros_cloudbuild" 62DEV_IMAGE_VERSION = (CROSVM_ROOT / "tools/impl/dev_container/version").read_text().strip() 63 64CACHE_DIR = os.environ.get("CROSVM_CONTAINER_CACHE", None) 65 66COMMON_ARGS = [ 67 # Share cache dir 68 f"--volume {CACHE_DIR}:/cache:rw" if CACHE_DIR else None, 69 # Use tmpfs in the container for faster performance. 70 "--mount type=tmpfs,destination=/tmp", 71 # KVM is required to run a VM for testing. 72 "--device /dev/kvm" if Path("/dev/kvm").is_char_device() else None, 73 # Enable terminal colors 74 f"--env TERM={os.environ.get('TERM', 'xterm-256color')}", 75] 76 77DOCKER_ARGS = [ 78 *COMMON_ARGS, 79] 80 81PODMAN_ARGS = [ 82 *COMMON_ARGS, 83 # Allow access to group permissions of the user (e.g. for kvm access). 84 "--group-add keep-groups" if os.name == "posix" else None, 85 # Increase number of PIDs the container can spawn (we run a lot of test processes in parallel) 86 "--pids-limit=4096" if os.name == "posix" else None, 87] 88 89# Environment variables to pass through to the container if they are specified. 90ENV_PASSTHROUGH = [ 91 "NEXTEST_PROFILE", 92 "http_proxy", 93 "https_proxy", 94] 95 96 97def machine_is_running(docker: cmd): 98 machine_state = docker("machine info").stdout() 99 return "MachineState: Running" in machine_state 100 101 102def container_name(cros: bool): 103 if cros: 104 return CROS_CONTAINER_NAME 105 else: 106 return DEV_CONTAINER_NAME 107 108 109def container_revision(docker: cmd, container_id: str): 110 image = docker("container inspect -f {{.Config.Image}}", container_id).stdout() 111 parts = image.split(":") 112 assert len(parts) == 2, f"Invalid image name {image}" 113 return parts[1] 114 115 116def container_id(docker: cmd, cros: bool): 117 return docker(f"ps -a -q -f name={container_name(cros)}").stdout() 118 119 120def container_is_running(docker: cmd, cros: bool): 121 return bool(docker(f"ps -q -f name={container_name(cros)}").stdout()) 122 123 124def delete_container(docker: cmd, cros: bool): 125 cid = container_id(docker, cros) 126 if cid: 127 print(f"Deleting dev-container {cid}.") 128 docker("rm -f", cid).fg(quiet=True) 129 return True 130 return False 131 132 133def workspace_mount_args(cros: bool): 134 """ 135 Returns arguments for mounting the crosvm sources to /workspace. 136 137 In ChromeOS checkouts the crosvm repo uses a symlink or worktree checkout, which links to a 138 different folder in the ChromeOS checkout. So we need to mount the whole CrOS checkout. 139 """ 140 if cros: 141 return ["--workdir /home/crosvmdev/chromiumos/src/platform/crosvm"] 142 elif is_cros_repo(): 143 return [ 144 f"--volume {quoted(cros_repo_root())}:/workspace:rw", 145 "--workdir /workspace/src/platform/crosvm", 146 ] 147 elif is_kiwi_repo(): 148 return [ 149 f"--volume {quoted(kiwi_repo_root())}:/workspace:rw", 150 # We override /scratch because we run out of memory if we use memory to back the 151 # `/scratch` mount point. 152 f"--volume {quoted(kiwi_repo_root())}/scratch:/scratch/cargo_target:rw", 153 "--workdir /workspace/platform/crosvm", 154 ] 155 elif is_aosp_repo(): 156 return [ 157 f"--volume {quoted(aosp_repo_root())}:/workspace:rw", 158 "--workdir /workspace/external/crosvm", 159 ] 160 else: 161 return [ 162 f"--volume {quoted(CROSVM_ROOT)}:/workspace:rw", 163 ] 164 165 166def ensure_container_is_alive(docker: cmd, docker_args: List[Optional[str]], cros: bool): 167 cid = container_id(docker, cros) 168 if cid and not container_is_running(docker, cros): 169 print("Existing container is not running.") 170 delete_container(docker, cros) 171 elif cid and not cros and container_revision(docker, cid) != DEV_IMAGE_VERSION: 172 print(f"New image is available.") 173 delete_container(docker, cros) 174 175 if not container_is_running(docker, cros): 176 # Run neverending sleep to keep container alive while we 'docker exec' commands. 177 print(f"Starting container...") 178 docker( 179 f"run --detach --name {container_name(cros)}", 180 *docker_args, 181 "sleep infinity", 182 ).fg(quiet=False) 183 cid = container_id(docker, cros) 184 else: 185 cid = container_id(docker, cros) 186 print(f"Using existing container ({cid}).") 187 return cid 188 189 190def validate_podman(podman: cmd): 191 graph_driver_name = podman("info --format={{.Store.GraphDriverName}}").stdout() 192 config_file_name = podman("info --format={{.Store.ConfigFile}}").stdout() 193 if graph_driver_name == "vfs": 194 print("You are using vfs as a storage driver. This will be extremely slow.") 195 print("Using the overlay driver is strongly recommended.") 196 print("Note: This will delete all existing podman images and containers.") 197 if confirm(f"Do you want me to update your config in {config_file_name}?"): 198 podman("system reset -f").fg() 199 with open(config_file_name, "a") as config_file: 200 print("[storage]", file=config_file) 201 print('driver = "overlay"', file=config_file) 202 203 if os.name == "posix": 204 username = os.environ["USER"] 205 subuids = Path("/etc/subuid").read_text() 206 if not username in subuids: 207 print("Rootless podman requires subuid's to be set up for your user.") 208 usermod = cmd( 209 "sudo usermod --add-subuids 900000-965535 --add-subgids 900000-965535", username 210 ) 211 print("I can fix that by running:", usermod) 212 if confirm("Ok?"): 213 usermod.fg() 214 podman("system migrate").fg() 215 216 217def main(argv: List[str]): 218 parser = argparse.ArgumentParser() 219 add_common_args(parser) 220 parser.add_argument("--stop", action="store_true") 221 parser.add_argument("--clean", action="store_true") 222 parser.add_argument("--hermetic", action="store_true") 223 parser.add_argument("--no-interactive", action="store_true") 224 parser.add_argument("--use-docker", action="store_true") 225 parser.add_argument("--self-test", action="store_true") 226 parser.add_argument("--pull", action="store_true") 227 parser.add_argument("--cros", action="store_true") 228 parser.add_argument("command", nargs=argparse.REMAINDER) 229 230 args = parser.parse_args(argv) 231 232 chdir(CROSVM_ROOT) 233 234 if CACHE_DIR: 235 Path(CACHE_DIR).mkdir(exist_ok=True) 236 237 has_docker = shutil.which("docker") != None 238 has_podman = shutil.which("podman") != None 239 if not has_podman and not has_docker: 240 raise Exception("Please install podman (or docker) to use the dev container.") 241 242 use_docker = args.use_docker 243 if has_docker and not has_podman: 244 use_docker = True 245 246 # cros container only works in docker 247 if args.cros: 248 use_docker = True 249 250 if use_docker: 251 print( 252 "WARNING: Running dev_container with docker may cause root-owned files to be created." 253 ) 254 print("Use podman to prevent this.") 255 print() 256 docker = cmd("docker") 257 docker_args = [ 258 *DOCKER_ARGS, 259 *workspace_mount_args(args.cros), 260 ] 261 else: 262 docker = cmd("podman") 263 264 # On windows, podman uses wsl vm. start the default podman vm for the rest of the script 265 # to work properly. 266 if os.name == "nt" and not machine_is_running(docker): 267 print("Starting podman default machine.") 268 docker("machine start").fg(quiet=True) 269 docker_args = [ 270 *PODMAN_ARGS, 271 *workspace_mount_args(args.cros), 272 ] 273 validate_podman(docker) 274 275 if args.cros: 276 docker_args.append("--privileged") # cros container requires privileged container 277 docker_args.append(CROS_IMAGE_NAME) 278 else: 279 docker_args.append(DEV_IMAGE_NAME + ":" + DEV_IMAGE_VERSION) 280 281 # Add environment variables to command line 282 exec_args: List[str] = [] 283 for key in ENV_PASSTHROUGH: 284 value = os.environ.get(key) 285 if value is not None: 286 exec_args.append("--env") 287 exec_args.append(f"{key}={quoted(value)}") 288 289 if args.self_test: 290 TestDevContainer.docker = docker 291 suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestDevContainer) 292 unittest.TextTestRunner().run(suite) 293 return 294 295 if args.stop: 296 if not delete_container(docker, args.cros): 297 print(f"container is not running.") 298 return 299 300 if args.clean: 301 delete_container(docker, args.cros) 302 303 if args.pull: 304 if args.cros: 305 docker("pull", CROS_IMAGE_NAME).fg() 306 else: 307 docker("pull", f"gcr.io/crosvm-infra/crosvm_dev:{DEV_IMAGE_VERSION}").fg() 308 return 309 310 command = args.command 311 312 # Default to interactive mode if a tty is present. 313 tty_args: List[str] = [] 314 if sys.stdin.isatty(): 315 tty_args += ["--tty"] 316 if not args.no_interactive: 317 tty_args += ["--interactive"] 318 319 # Start an interactive shell by default 320 if args.hermetic: 321 # cmd is passed to entrypoint 322 quoted_cmd = list(map(quoted, command)) 323 docker(f"run --rm", *tty_args, *docker_args, *exec_args, *quoted_cmd).fg() 324 else: 325 # cmd is executed directly 326 cid = ensure_container_is_alive(docker, docker_args, args.cros) 327 if not command: 328 command = ("/bin/bash",) 329 quoted_cmd = list(map(quoted, command)) 330 docker("exec", *tty_args, *exec_args, cid, *quoted_cmd).fg() 331 332 333class TestDevContainer(unittest.TestCase): 334 """ 335 Runs live tests using the docker service. 336 337 Note: This test is not run by health-check since it cannot be run inside the 338 container. It is run by infra/recipes/health_check.py before running health checks. 339 """ 340 341 docker: cmd 342 docker_args = [ 343 *workspace_mount_args(cros=False), 344 *DOCKER_ARGS, 345 ] 346 347 def setUp(self): 348 # Start with a stopped container for each test. 349 delete_container(self.docker, cros=False) 350 351 def test_stopped_container(self): 352 # Create but do not run a new container. 353 self.docker( 354 f"create --name {DEV_CONTAINER_NAME}", *self.docker_args, "sleep infinity" 355 ).stdout() 356 self.assertTrue(container_id(self.docker, cros=False)) 357 self.assertFalse(container_is_running(self.docker, cros=False)) 358 359 def test_container_reuse(self): 360 cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False) 361 cid2 = ensure_container_is_alive(self.docker, self.docker_args, cros=False) 362 self.assertEqual(cid, cid2) 363 364 def test_handling_of_stopped_container(self): 365 cid = ensure_container_is_alive(self.docker, self.docker_args, cros=False) 366 self.docker("kill", cid).fg() 367 368 # Make sure we can get back into a good state and execute commands. 369 ensure_container_is_alive(self.docker, self.docker_args, cros=False) 370 self.assertTrue(container_is_running(self.docker, cros=False)) 371 main(["true"]) 372 373 374if __name__ == "__main__": 375 main(sys.argv[1:]) 376