1 // Copyright 2017 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 use std::cell::RefCell;
6 use std::fs::File;
7 use std::io::prelude::*;
8 use std::process;
9 use std::sync::mpsc;
10 use std::sync::Arc;
11 use std::sync::Barrier;
12 use std::thread;
13 use std::thread::JoinHandle;
14 #[cfg(target_arch = "x86_64")]
15 use std::time::Duration;
16
17 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
18 use aarch64::AArch64 as Arch;
19 use anyhow::Context;
20 use anyhow::Result;
21 use arch::CpuConfigArch;
22 use arch::CpuSet;
23 use arch::IrqChipArch;
24 use arch::LinuxArch;
25 use arch::VcpuArch;
26 use arch::VcpuInitArch;
27 use arch::VmArch;
28 use base::gettid;
29 use base::sched_attr;
30 use base::sched_setattr;
31 use base::signal::clear_signal_handler;
32 use base::signal::BlockedSignal;
33 use base::*;
34 use devices::Bus;
35 use devices::IrqChip;
36 use devices::VcpuRunState;
37 use hypervisor::IoOperation;
38 use hypervisor::IoParams;
39 use hypervisor::VcpuExit;
40 use hypervisor::VcpuSignalHandle;
41 use libc::c_int;
42 use metrics_events::MetricEventType;
43 #[cfg(target_arch = "riscv64")]
44 use riscv64::Riscv64 as Arch;
45 use serde::Deserialize;
46 use serde::Serialize;
47 #[cfg(target_arch = "x86_64")]
48 use sync::Mutex;
49 use vm_control::*;
50 #[cfg(feature = "gdb")]
51 use vm_memory::GuestMemory;
52 #[cfg(target_arch = "x86_64")]
53 use x86_64::X8664arch as Arch;
54
55 use super::ExitState;
56 #[cfg(target_arch = "x86_64")]
57 use crate::crosvm::ratelimit::Ratelimit;
58
59 // TODO(davidai): Import libc constant when updated
60 const SCHED_FLAG_RESET_ON_FORK: u64 = 0x1;
61 const SCHED_FLAG_KEEP_POLICY: u64 = 0x08;
62 const SCHED_FLAG_KEEP_PARAMS: u64 = 0x10;
63 const SCHED_FLAG_UTIL_CLAMP_MIN: u64 = 0x20;
64 const SCHED_SCALE_CAPACITY: u32 = 1024;
65 const SCHED_FLAG_KEEP_ALL: u64 = SCHED_FLAG_KEEP_POLICY | SCHED_FLAG_KEEP_PARAMS;
66
67 /// Set the VCPU thread affinity and other per-thread scheduler properties.
68 /// This function will be called from each VCPU thread at startup.
69 #[allow(clippy::unnecessary_cast)]
set_vcpu_thread_scheduling( vcpu_affinity: CpuSet, core_scheduling: bool, enable_per_vm_core_scheduling: bool, vcpu_cgroup_tasks_file: Option<File>, run_rt: bool, boost_uclamp: bool, ) -> anyhow::Result<()>70 pub fn set_vcpu_thread_scheduling(
71 vcpu_affinity: CpuSet,
72 core_scheduling: bool,
73 enable_per_vm_core_scheduling: bool,
74 vcpu_cgroup_tasks_file: Option<File>,
75 run_rt: bool,
76 boost_uclamp: bool,
77 ) -> anyhow::Result<()> {
78 if boost_uclamp {
79 let mut sched_attr = sched_attr::default();
80 sched_attr.sched_flags = SCHED_FLAG_KEEP_ALL as u64
81 | SCHED_FLAG_UTIL_CLAMP_MIN
82 | SCHED_FLAG_RESET_ON_FORK as u64;
83 sched_attr.sched_util_min = SCHED_SCALE_CAPACITY;
84
85 if let Err(e) = sched_setattr(0, &mut sched_attr, 0) {
86 warn!("Failed to boost vcpu util: {}", e);
87 }
88 }
89
90 if core_scheduling && !enable_per_vm_core_scheduling {
91 // Do per-vCPU core scheduling by setting a unique cookie to each vCPU.
92 if let Err(e) = enable_core_scheduling() {
93 error!("Failed to enable core scheduling: {}", e);
94 }
95 }
96
97 // Move vcpu thread to cgroup
98 if let Some(mut f) = vcpu_cgroup_tasks_file {
99 f.write_all(base::gettid().to_string().as_bytes())
100 .context("failed to write vcpu tid to cgroup tasks")?;
101 }
102
103 // vcpu_affinity needs to be set after moving to cgroup
104 // or it will be overriden by cgroup settings, vcpu_affinity
105 // here is bounded by the cpuset specified in the cgroup
106 if !vcpu_affinity.is_empty() {
107 if let Err(e) = set_cpu_affinity(vcpu_affinity) {
108 error!("Failed to set CPU affinity: {}", e);
109 }
110 }
111
112 if run_rt {
113 const DEFAULT_VCPU_RT_LEVEL: u16 = 6;
114 if let Err(e) = set_rt_prio_limit(u64::from(DEFAULT_VCPU_RT_LEVEL))
115 .and_then(|_| set_rt_round_robin(i32::from(DEFAULT_VCPU_RT_LEVEL)))
116 {
117 warn!("Failed to set vcpu to real time: {}", e);
118 }
119 }
120
121 Ok(())
122 }
123
124 // Sets up a vcpu and converts it into a runnable vcpu.
runnable_vcpu<V>( cpu_id: usize, vcpu_id: usize, vcpu: Option<V>, vcpu_init: VcpuInitArch, vm: impl VmArch, irq_chip: &mut dyn IrqChipArch, vcpu_count: usize, cpu_config: Option<CpuConfigArch>, ) -> Result<V> where V: VcpuArch,125 pub fn runnable_vcpu<V>(
126 cpu_id: usize,
127 vcpu_id: usize,
128 vcpu: Option<V>,
129 vcpu_init: VcpuInitArch,
130 vm: impl VmArch,
131 irq_chip: &mut dyn IrqChipArch,
132 vcpu_count: usize,
133 cpu_config: Option<CpuConfigArch>,
134 ) -> Result<V>
135 where
136 V: VcpuArch,
137 {
138 let mut vcpu = match vcpu {
139 Some(v) => v,
140 None => {
141 // If vcpu is None, it means this arch/hypervisor requires create_vcpu to be called from
142 // the vcpu thread.
143 match vm
144 .create_vcpu(vcpu_id)
145 .context("failed to create vcpu")?
146 .downcast::<V>()
147 {
148 Ok(v) => *v,
149 Err(_) => panic!("VM created wrong type of VCPU"),
150 }
151 }
152 };
153
154 irq_chip
155 .add_vcpu(cpu_id, &vcpu)
156 .context("failed to add vcpu to irq chip")?;
157
158 Arch::configure_vcpu(
159 &vm,
160 vm.get_hypervisor(),
161 irq_chip,
162 &mut vcpu,
163 vcpu_init,
164 cpu_id,
165 vcpu_count,
166 cpu_config,
167 )
168 .context("failed to configure vcpu")?;
169
170 Ok(vcpu)
171 }
172
173 thread_local!(static VCPU_THREAD: RefCell<Option<VcpuSignalHandle>> = const { RefCell::new(None) });
174
set_vcpu_thread_local(vcpu: Option<&dyn VcpuArch>, signal_num: c_int)175 fn set_vcpu_thread_local(vcpu: Option<&dyn VcpuArch>, signal_num: c_int) {
176 // Block signal while we add -- if a signal fires (very unlikely,
177 // as this means something is trying to pause the vcpu before it has
178 // even started) it'll try to grab the read lock while this write
179 // lock is grabbed and cause a deadlock.
180 // Assuming that a failure to block means it's already blocked.
181 let _blocked_signal = BlockedSignal::new(signal_num);
182
183 VCPU_THREAD.with(|v| {
184 let mut vcpu_thread = v.borrow_mut();
185
186 if let Some(vcpu) = vcpu {
187 assert!(vcpu_thread.is_none());
188 *vcpu_thread = Some(vcpu.signal_handle());
189 } else {
190 *vcpu_thread = None;
191 }
192 });
193 }
194
setup_vcpu_signal_handler() -> Result<()>195 pub fn setup_vcpu_signal_handler() -> Result<()> {
196 // SAFETY: trivially safe as we check return value.
197 unsafe {
198 extern "C" fn handle_signal(_: c_int) {
199 // Use LocalKey::try_with() so we don't panic if a signal happens while the destructor
200 // is running, and ignore any errors (the assumption being that the thread is exiting
201 // anyway in that case).
202 let _result = VCPU_THREAD.try_with(|v| {
203 if let Some(vcpu_signal_handle) = &(*v.borrow()) {
204 vcpu_signal_handle.signal_immediate_exit();
205 }
206 });
207 }
208
209 register_rt_signal_handler(SIGRTMIN() + 0, handle_signal)
210 .context("error registering signal handler")?;
211 }
212 Ok(())
213 }
214
remove_vcpu_signal_handler() -> Result<()>215 pub fn remove_vcpu_signal_handler() -> Result<()> {
216 clear_signal_handler(SIGRTMIN() + 0).context("error unregistering signal handler")
217 }
218
vcpu_loop<V>( mut run_mode: VmRunMode, cpu_id: usize, mut vcpu: V, irq_chip: Box<dyn IrqChipArch + 'static>, run_rt: bool, delay_rt: bool, io_bus: Bus, mmio_bus: Bus, from_main_tube: mpsc::Receiver<VcpuControl>, #[cfg(feature = "gdb")] to_gdb_tube: Option<mpsc::Sender<VcpuDebugStatusMessage>>, #[cfg(feature = "gdb")] guest_mem: GuestMemory, #[cfg(target_arch = "x86_64")] bus_lock_ratelimit_ctrl: Arc<Mutex<Ratelimit>>, ) -> ExitState where V: VcpuArch,219 fn vcpu_loop<V>(
220 mut run_mode: VmRunMode,
221 cpu_id: usize,
222 mut vcpu: V,
223 irq_chip: Box<dyn IrqChipArch + 'static>,
224 run_rt: bool,
225 delay_rt: bool,
226 io_bus: Bus,
227 mmio_bus: Bus,
228 from_main_tube: mpsc::Receiver<VcpuControl>,
229 #[cfg(feature = "gdb")] to_gdb_tube: Option<mpsc::Sender<VcpuDebugStatusMessage>>,
230 #[cfg(feature = "gdb")] guest_mem: GuestMemory,
231 #[cfg(target_arch = "x86_64")] bus_lock_ratelimit_ctrl: Arc<Mutex<Ratelimit>>,
232 ) -> ExitState
233 where
234 V: VcpuArch,
235 {
236 let mut interrupted_by_signal = false;
237
238 loop {
239 // Start by checking for messages to process and the run state of the CPU.
240 // An extra check here for Running so there isn't a need to call recv unless a
241 // message is likely to be ready because a signal was sent.
242 if interrupted_by_signal || run_mode != VmRunMode::Running {
243 'state_loop: loop {
244 // Tries to get a pending message without blocking first.
245 let msg = match from_main_tube.try_recv() {
246 Ok(m) => m,
247 Err(mpsc::TryRecvError::Empty) if run_mode == VmRunMode::Running => {
248 // If the VM is running and no message is pending, the state won't
249 // change.
250 break 'state_loop;
251 }
252 Err(mpsc::TryRecvError::Empty) => {
253 // If the VM is not running, wait until a message is ready.
254 match from_main_tube.recv() {
255 Ok(m) => m,
256 Err(mpsc::RecvError) => {
257 error!("Failed to read from main tube in vcpu");
258 return ExitState::Crash;
259 }
260 }
261 }
262 Err(mpsc::TryRecvError::Disconnected) => {
263 error!("Failed to read from main tube in vcpu");
264 return ExitState::Crash;
265 }
266 };
267
268 // Collect all pending messages.
269 let mut messages = vec![msg];
270 messages.append(&mut from_main_tube.try_iter().collect());
271
272 for msg in messages {
273 match msg {
274 VcpuControl::RunState(new_mode) => {
275 run_mode = new_mode;
276 match run_mode {
277 VmRunMode::Running => {}
278 VmRunMode::Suspending => {
279 if let Err(e) = vcpu.on_suspend() {
280 error!(
281 "failed to tell hypervisor vcpu {} is suspending: {}",
282 cpu_id, e
283 );
284 }
285 }
286 VmRunMode::Breakpoint => {}
287 VmRunMode::Exiting => return ExitState::Stop,
288 }
289 }
290 #[cfg(feature = "gdb")]
291 VcpuControl::Debug(d) => {
292 if let Err(e) = crate::crosvm::gdb::vcpu_control_debug(
293 cpu_id,
294 &vcpu,
295 &guest_mem,
296 d,
297 to_gdb_tube.as_ref(),
298 ) {
299 error!("Failed to handle VcpuControl::Debug message: {:#}", e);
300 }
301 }
302 VcpuControl::MakeRT => {
303 if run_rt && delay_rt {
304 info!("Making vcpu {} RT\n", cpu_id);
305 const DEFAULT_VCPU_RT_LEVEL: u16 = 6;
306 if let Err(e) = set_rt_prio_limit(u64::from(DEFAULT_VCPU_RT_LEVEL))
307 .and_then(|_| {
308 set_rt_round_robin(i32::from(DEFAULT_VCPU_RT_LEVEL))
309 })
310 {
311 warn!("Failed to set vcpu to real time: {}", e);
312 }
313 }
314 }
315 VcpuControl::GetStates(response_chan) => {
316 if let Err(e) = response_chan.send(run_mode) {
317 error!("Failed to send GetState: {}", e);
318 };
319 }
320 VcpuControl::Snapshot(snapshot_writer, response_chan) => {
321 let resp = vcpu
322 .snapshot()
323 .and_then(|s| {
324 snapshot_writer
325 .write_fragment(&format!("vcpu{}", vcpu.id()), &s)
326 })
327 .with_context(|| format!("Failed to snapshot Vcpu #{}", vcpu.id()));
328 if let Err(e) = response_chan.send(resp) {
329 error!("Failed to send snapshot response: {}", e);
330 }
331 }
332 VcpuControl::Restore(req) => {
333 let resp = req
334 .snapshot_reader
335 .read_fragment(&format!("vcpu{}", vcpu.id()))
336 .and_then(|s| {
337 vcpu.restore(
338 &s,
339 #[cfg(target_arch = "x86_64")]
340 req.host_tsc_reference_moment,
341 )
342 })
343 .with_context(|| format!("Failed to restore Vcpu #{}", vcpu.id()));
344 if let Err(e) = req.result_sender.send(resp) {
345 error!("Failed to send restore response: {}", e);
346 }
347 }
348 VcpuControl::Throttle(target_us) => {
349 let start_time = std::time::Instant::now();
350
351 while start_time.elapsed().as_micros() < target_us.into() {
352 // TODO: Investigate replacing this with std::hint::spin_loop()
353 // to hint to the pCPU to potentially save some power. Also revisit
354 // this when scheduler updates are available on newer kernel
355 // versions.
356 }
357 }
358 }
359 }
360 if run_mode == VmRunMode::Running {
361 break 'state_loop;
362 }
363 }
364 }
365
366 interrupted_by_signal = false;
367
368 // Vcpus may have run a HLT instruction, which puts them into a state other than
369 // VcpuRunState::Runnable. In that case, this call to wait_until_runnable blocks
370 // until either the irqchip receives an interrupt for this vcpu, or until the main
371 // thread kicks this vcpu as a result of some VmControl operation. In most IrqChip
372 // implementations HLT instructions do not make it to crosvm, and thus this is a
373 // no-op that always returns VcpuRunState::Runnable.
374 match irq_chip.wait_until_runnable(vcpu.as_vcpu()) {
375 Ok(VcpuRunState::Runnable) => {}
376 Ok(VcpuRunState::Interrupted) => interrupted_by_signal = true,
377 Err(e) => error!(
378 "error waiting for vcpu {} to become runnable: {}",
379 cpu_id, e
380 ),
381 }
382
383 if !interrupted_by_signal {
384 match vcpu.run() {
385 Ok(VcpuExit::Io) => {
386 if let Err(e) =
387 vcpu.handle_io(&mut |IoParams { address, operation }| match operation {
388 IoOperation::Read(data) => {
389 io_bus.read(address, data);
390 }
391 IoOperation::Write(data) => {
392 io_bus.write(address, data);
393 }
394 })
395 {
396 error!("failed to handle io: {}", e)
397 }
398 }
399 Ok(VcpuExit::Mmio) => {
400 if let Err(e) =
401 vcpu.handle_mmio(&mut |IoParams { address, operation }| match operation {
402 IoOperation::Read(data) => {
403 mmio_bus.read(address, data);
404 Ok(())
405 }
406 IoOperation::Write(data) => {
407 mmio_bus.write(address, data);
408 Ok(())
409 }
410 })
411 {
412 error!("failed to handle mmio: {}", e);
413 }
414 }
415 Ok(VcpuExit::IoapicEoi { vector }) => {
416 if let Err(e) = irq_chip.broadcast_eoi(vector) {
417 error!(
418 "failed to broadcast eoi {} on vcpu {}: {}",
419 vector, cpu_id, e
420 );
421 }
422 }
423 Ok(VcpuExit::IrqWindowOpen) => {}
424 Ok(VcpuExit::Hlt) => irq_chip.halted(cpu_id),
425 Ok(VcpuExit::Shutdown(reason)) => {
426 if let Err(e) = reason {
427 metrics::log_descriptor(
428 MetricEventType::VcpuShutdownError,
429 e.get_raw_error_code() as i64,
430 );
431 }
432 return ExitState::Stop;
433 }
434 Ok(VcpuExit::FailEntry {
435 hardware_entry_failure_reason,
436 }) => {
437 error!("vcpu hw run failure: {:#x}", hardware_entry_failure_reason);
438 return ExitState::Crash;
439 }
440 Ok(VcpuExit::SystemEventShutdown) => {
441 info!("system shutdown event on vcpu {}", cpu_id);
442 return ExitState::Stop;
443 }
444 Ok(VcpuExit::SystemEventReset) => {
445 info!("system reset event");
446 return ExitState::Reset;
447 }
448 Ok(VcpuExit::SystemEventCrash) => {
449 info!("system crash event on vcpu {}", cpu_id);
450 return ExitState::Stop;
451 }
452 Ok(VcpuExit::Debug) => {
453 #[cfg(feature = "gdb")]
454 if let Err(e) =
455 crate::crosvm::gdb::vcpu_exit_debug(cpu_id, to_gdb_tube.as_ref())
456 {
457 error!("Failed to handle VcpuExit::Debug: {:#}", e);
458 return ExitState::Crash;
459 }
460
461 run_mode = VmRunMode::Breakpoint;
462 }
463 #[cfg(target_arch = "x86_64")]
464 Ok(VcpuExit::BusLock) => {
465 let delay_ns: u64 = bus_lock_ratelimit_ctrl.lock().ratelimit_calculate_delay(1);
466 thread::sleep(Duration::from_nanos(delay_ns));
467 }
468 Ok(VcpuExit::Sbi {
469 extension_id: _,
470 function_id: _,
471 args: _,
472 }) => {
473 unimplemented!("Sbi exits not yet supported");
474 }
475 Ok(VcpuExit::RiscvCsr {
476 csr_num,
477 new_value,
478 write_mask,
479 ret_value: _,
480 }) => {
481 unimplemented!(
482 "csr exit! {:#x} to {:#x} mask {:#x}",
483 csr_num,
484 new_value,
485 write_mask
486 );
487 }
488
489 Ok(r) => warn!("unexpected vcpu exit: {:?}", r),
490 Err(e) => match e.errno() {
491 libc::EINTR => interrupted_by_signal = true,
492 libc::EAGAIN => {}
493 _ => {
494 error!("vcpu hit unknown error: {}", e);
495 return ExitState::Crash;
496 }
497 },
498 }
499 }
500
501 if interrupted_by_signal {
502 vcpu.set_immediate_exit(false);
503 }
504
505 if let Err(e) = irq_chip.inject_interrupts(vcpu.as_vcpu()) {
506 error!("failed to inject interrupts for vcpu {}: {}", cpu_id, e);
507 }
508 }
509 }
510
511 #[derive(Serialize, Deserialize)]
512 pub struct VcpuPidTid {
513 pub vcpu_id: usize,
514 pub process_id: u32,
515 pub thread_id: u32,
516 }
517
run_vcpu<V>( cpu_id: usize, vcpu_id: usize, vcpu: Option<V>, vcpu_init: VcpuInitArch, vm: impl VmArch + 'static, mut irq_chip: Box<dyn IrqChipArch + 'static>, vcpu_count: usize, run_rt: bool, vcpu_affinity: CpuSet, delay_rt: bool, start_barrier: Arc<Barrier>, mut io_bus: Bus, mut mmio_bus: Bus, vm_evt_wrtube: SendTube, from_main_tube: mpsc::Receiver<VcpuControl>, #[cfg(feature = "gdb")] to_gdb_tube: Option<mpsc::Sender<VcpuDebugStatusMessage>>, enable_core_scheduling: bool, enable_per_vm_core_scheduling: bool, cpu_config: Option<CpuConfigArch>, vcpu_cgroup_tasks_file: Option<File>, #[cfg(target_arch = "x86_64")] bus_lock_ratelimit_ctrl: Arc<Mutex<Ratelimit>>, run_mode: VmRunMode, boost_uclamp: bool, vcpu_pid_tid_tube: mpsc::Sender<VcpuPidTid>, ) -> Result<JoinHandle<()>> where V: VcpuArch + 'static,518 pub fn run_vcpu<V>(
519 cpu_id: usize,
520 vcpu_id: usize,
521 vcpu: Option<V>,
522 vcpu_init: VcpuInitArch,
523 vm: impl VmArch + 'static,
524 mut irq_chip: Box<dyn IrqChipArch + 'static>,
525 vcpu_count: usize,
526 run_rt: bool,
527 vcpu_affinity: CpuSet,
528 delay_rt: bool,
529 start_barrier: Arc<Barrier>,
530 mut io_bus: Bus,
531 mut mmio_bus: Bus,
532 vm_evt_wrtube: SendTube,
533 from_main_tube: mpsc::Receiver<VcpuControl>,
534 #[cfg(feature = "gdb")] to_gdb_tube: Option<mpsc::Sender<VcpuDebugStatusMessage>>,
535 enable_core_scheduling: bool,
536 enable_per_vm_core_scheduling: bool,
537 cpu_config: Option<CpuConfigArch>,
538 vcpu_cgroup_tasks_file: Option<File>,
539 #[cfg(target_arch = "x86_64")] bus_lock_ratelimit_ctrl: Arc<Mutex<Ratelimit>>,
540 run_mode: VmRunMode,
541 boost_uclamp: bool,
542 vcpu_pid_tid_tube: mpsc::Sender<VcpuPidTid>,
543 ) -> Result<JoinHandle<()>>
544 where
545 V: VcpuArch + 'static,
546 {
547 thread::Builder::new()
548 .name(format!("crosvm_vcpu{}", cpu_id))
549 .spawn(move || {
550 // Having a closure returning ExitState guarentees that we
551 // send a VmEventType on all code paths after the closure
552 // returns.
553 let vcpu_fn = || -> ExitState {
554 if let Err(e) = set_vcpu_thread_scheduling(
555 vcpu_affinity,
556 enable_core_scheduling,
557 enable_per_vm_core_scheduling,
558 vcpu_cgroup_tasks_file,
559 run_rt && !delay_rt,
560 boost_uclamp,
561 ) {
562 error!("vcpu thread setup failed: {:#}", e);
563 return ExitState::Stop;
564 }
565
566 if let Err(e) = vcpu_pid_tid_tube.send(VcpuPidTid {
567 vcpu_id: cpu_id,
568 process_id: process::id(),
569 thread_id: gettid() as u32,
570 }) {
571 error!("Failed to send vcpu process/thread id: {:#}", e);
572 return ExitState::Crash;
573 }
574
575 #[cfg(feature = "gdb")]
576 let guest_mem = vm.get_memory().clone();
577
578 let runnable_vcpu = runnable_vcpu(
579 cpu_id,
580 vcpu_id,
581 vcpu,
582 vcpu_init,
583 vm,
584 irq_chip.as_mut(),
585 vcpu_count,
586 cpu_config,
587 );
588
589 start_barrier.wait();
590
591 let vcpu = match runnable_vcpu {
592 Ok(v) => v,
593 Err(e) => {
594 error!("failed to start vcpu {}: {:#}", cpu_id, e);
595 return ExitState::Stop;
596 }
597 };
598
599 set_vcpu_thread_local(Some(&vcpu), SIGRTMIN() + 0);
600
601 mmio_bus.set_access_id(cpu_id);
602 io_bus.set_access_id(cpu_id);
603
604 let vcpu_exit_state = vcpu_loop(
605 run_mode,
606 cpu_id,
607 vcpu,
608 irq_chip,
609 run_rt,
610 delay_rt,
611 io_bus,
612 mmio_bus,
613 from_main_tube,
614 #[cfg(feature = "gdb")]
615 to_gdb_tube,
616 #[cfg(feature = "gdb")]
617 guest_mem,
618 #[cfg(target_arch = "x86_64")]
619 bus_lock_ratelimit_ctrl,
620 );
621
622 // We don't want any more VCPU signals from now until the thread exits.
623 let _ = block_signal(SIGRTMIN() + 0);
624 set_vcpu_thread_local(None, SIGRTMIN() + 0);
625
626 vcpu_exit_state
627 };
628
629 let final_event_data = match vcpu_fn() {
630 ExitState::Stop => VmEventType::Exit,
631 ExitState::Reset => VmEventType::Reset,
632 ExitState::Crash => VmEventType::Crash,
633 // vcpu_loop doesn't exit with GuestPanic.
634 ExitState::GuestPanic => unreachable!(),
635 ExitState::WatchdogReset => VmEventType::WatchdogReset,
636 };
637 if let Err(e) = vm_evt_wrtube.send::<VmEventType>(&final_event_data) {
638 error!(
639 "failed to send final event {:?} on vcpu {}: {}",
640 final_event_data, cpu_id, e
641 )
642 }
643 })
644 .context("failed to spawn VCPU thread")
645 }
646
647 /// Signals all running VCPUs to vmexit, sends VcpuControl message to each VCPU tube, and tells
648 /// `irq_chip` to stop blocking halted VCPUs. The channel message is set first because both the
649 /// signal and the irq_chip kick could cause the VCPU thread to continue through the VCPU run
650 /// loop.
kick_all_vcpus( vcpu_handles: &[(JoinHandle<()>, mpsc::Sender<vm_control::VcpuControl>)], irq_chip: &dyn IrqChip, message: VcpuControl, )651 pub fn kick_all_vcpus(
652 vcpu_handles: &[(JoinHandle<()>, mpsc::Sender<vm_control::VcpuControl>)],
653 irq_chip: &dyn IrqChip,
654 message: VcpuControl,
655 ) {
656 for (handle, tube) in vcpu_handles {
657 if let Err(e) = tube.send(message.clone()) {
658 error!("failed to send VcpuControl: {}", e);
659 }
660 let _ = handle.kill(SIGRTMIN() + 0);
661 }
662 irq_chip.kick_halted_vcpus();
663 }
664
665 /// Signals specific running VCPUs to vmexit, sends VcpuControl message to the VCPU tube, and tells
666 /// `irq_chip` to stop blocking halted VCPUs. The channel message is set first because both the
667 /// signal and the irq_chip kick could cause the VCPU thread to continue through the VCPU run
668 /// loop.
kick_vcpu( vcpu_handle: &Option<&(JoinHandle<()>, mpsc::Sender<vm_control::VcpuControl>)>, irq_chip: &dyn IrqChip, message: VcpuControl, )669 pub fn kick_vcpu(
670 vcpu_handle: &Option<&(JoinHandle<()>, mpsc::Sender<vm_control::VcpuControl>)>,
671 irq_chip: &dyn IrqChip,
672 message: VcpuControl,
673 ) {
674 if let Some((handle, tube)) = vcpu_handle {
675 if let Err(e) = tube.send(message) {
676 error!("failed to send VcpuControl: {}", e);
677 }
678 let _ = handle.kill(SIGRTMIN() + 0);
679 }
680 irq_chip.kick_halted_vcpus();
681 }
682