1 // Copyright 2017 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #![cfg_attr(windows, allow(unused))]
6
7 //! Emulates virtual and hardware devices.
8
9 pub mod ac_adapter;
10 pub mod acpi;
11 pub mod bat;
12 mod bus;
13 #[cfg(feature = "stats")]
14 mod bus_stats;
15 pub mod cmos;
16 #[cfg(target_arch = "x86_64")]
17 mod debugcon;
18 mod fw_cfg;
19 mod i8042;
20 mod irq_event;
21 pub mod irqchip;
22 mod pci;
23 mod pflash;
24 pub mod pl030;
25 pub mod pmc_virt;
26 mod serial;
27 pub mod serial_device;
28 mod suspendable;
29 mod sys;
30 #[cfg(any(target_os = "android", target_os = "linux"))]
31 mod virtcpufreq;
32 #[cfg(any(target_os = "android", target_os = "linux"))]
33 mod virtcpufreq_v2;
34 pub mod virtio;
35 #[cfg(feature = "vtpm")]
36 mod vtpm_proxy;
37
38 cfg_if::cfg_if! {
39 if #[cfg(target_arch = "x86_64")] {
40 mod pit;
41 pub use self::pit::{Pit, PitError};
42 pub mod tsc;
43 }
44 }
45
46 use std::sync::Arc;
47
48 use anyhow::anyhow;
49 use anyhow::Context;
50 use base::debug;
51 use base::error;
52 use base::info;
53 use base::Tube;
54 use base::TubeError;
55 use cros_async::AsyncTube;
56 use cros_async::Executor;
57 use serde::Deserialize;
58 use serde::Serialize;
59 use vm_control::DeviceControlCommand;
60 use vm_control::DevicesState;
61 use vm_control::VmResponse;
62 use vm_memory::GuestMemory;
63
64 pub use self::acpi::ACPIPMFixedEvent;
65 pub use self::acpi::ACPIPMResource;
66 pub use self::bat::BatteryError;
67 pub use self::bat::GoldfishBattery;
68 pub use self::bus::Bus;
69 pub use self::bus::BusAccessInfo;
70 pub use self::bus::BusDevice;
71 pub use self::bus::BusDeviceObj;
72 pub use self::bus::BusDeviceSync;
73 pub use self::bus::BusRange;
74 pub use self::bus::BusResumeDevice;
75 pub use self::bus::BusType;
76 pub use self::bus::Error as BusError;
77 pub use self::bus::HotPlugBus;
78 pub use self::bus::HotPlugKey;
79 #[cfg(feature = "stats")]
80 pub use self::bus_stats::BusStatistics;
81 #[cfg(target_arch = "x86_64")]
82 pub use self::debugcon::Debugcon;
83 pub use self::fw_cfg::Error as FwCfgError;
84 pub use self::fw_cfg::FwCfgDevice;
85 pub use self::fw_cfg::FwCfgItemType;
86 pub use self::fw_cfg::FwCfgParameters;
87 pub use self::fw_cfg::FW_CFG_BASE_PORT;
88 pub use self::fw_cfg::FW_CFG_MAX_FILE_SLOTS;
89 pub use self::fw_cfg::FW_CFG_WIDTH;
90 pub use self::i8042::I8042Device;
91 pub use self::irq_event::IrqEdgeEvent;
92 pub use self::irq_event::IrqLevelEvent;
93 pub use self::irqchip::*;
94 pub use self::pci::BarRange;
95 pub use self::pci::CrosvmDeviceId;
96 pub use self::pci::GpeScope;
97 #[cfg(feature = "pci-hotplug")]
98 pub use self::pci::HotPluggable;
99 #[cfg(feature = "pci-hotplug")]
100 pub use self::pci::IntxParameter;
101 #[cfg(feature = "pci-hotplug")]
102 pub use self::pci::NetResourceCarrier;
103 pub use self::pci::PciAddress;
104 pub use self::pci::PciAddressError;
105 pub use self::pci::PciBarConfiguration;
106 pub use self::pci::PciBarIndex;
107 pub use self::pci::PciBus;
108 pub use self::pci::PciClassCode;
109 pub use self::pci::PciConfigIo;
110 pub use self::pci::PciConfigMmio;
111 pub use self::pci::PciDevice;
112 pub use self::pci::PciDeviceError;
113 pub use self::pci::PciInterruptPin;
114 pub use self::pci::PciMmioMapper;
115 pub use self::pci::PciRoot;
116 pub use self::pci::PciRootCommand;
117 pub use self::pci::PciVirtualConfigMmio;
118 pub use self::pci::PreferredIrq;
119 #[cfg(feature = "pci-hotplug")]
120 pub use self::pci::ResourceCarrier;
121 pub use self::pci::StubPciDevice;
122 pub use self::pci::StubPciParameters;
123 pub use self::pflash::Pflash;
124 pub use self::pflash::PflashParameters;
125 pub use self::pl030::Pl030;
126 pub use self::pmc_virt::VirtualPmc;
127 pub use self::serial::Serial;
128 pub use self::serial_device::Error as SerialError;
129 pub use self::serial_device::SerialDevice;
130 pub use self::serial_device::SerialHardware;
131 pub use self::serial_device::SerialParameters;
132 pub use self::serial_device::SerialType;
133 pub use self::suspendable::DeviceState;
134 pub use self::suspendable::Suspendable;
135 #[cfg(any(target_os = "android", target_os = "linux"))]
136 pub use self::virtcpufreq::VirtCpufreq;
137 #[cfg(any(target_os = "android", target_os = "linux"))]
138 pub use self::virtcpufreq_v2::VirtCpufreqV2;
139 pub use self::virtio::VirtioMmioDevice;
140 pub use self::virtio::VirtioPciDevice;
141 #[cfg(feature = "vtpm")]
142 pub use self::vtpm_proxy::VtpmProxy;
143
144 cfg_if::cfg_if! {
145 if #[cfg(any(target_os = "android", target_os = "linux"))] {
146 mod platform;
147 mod proxy;
148 pub mod vmwdt;
149 pub mod vfio;
150 #[cfg(feature = "usb")]
151 #[macro_use]
152 mod register_space;
153 #[cfg(feature = "usb")]
154 pub mod usb;
155 #[cfg(feature = "usb")]
156 mod utils;
157
158 pub use self::pci::{
159 CoIommuDev, CoIommuParameters, CoIommuUnpinPolicy, PciBridge, PcieDownstreamPort,
160 PcieHostPort, PcieRootPort, PcieUpstreamPort, PvPanicCode, PvPanicPciDevice,
161 VfioPciDevice,
162 };
163 pub use self::platform::VfioPlatformDevice;
164 pub use self::ac_adapter::AcAdapter;
165 pub use self::proxy::ChildProcIntf;
166 pub use self::proxy::Error as ProxyError;
167 pub use self::proxy::ProxyDevice;
168 #[cfg(feature = "usb")]
169 pub use self::usb::backend::device_provider::DeviceProvider;
170 #[cfg(feature = "usb")]
171 pub use self::usb::xhci::xhci_controller::XhciController;
172 pub use self::vfio::VfioContainer;
173 pub use self::vfio::VfioDevice;
174 pub use self::vfio::VfioDeviceType;
175 pub use self::virtio::vfio_wrapper;
176
177 } else if #[cfg(windows)] {
178 } else {
179 compile_error!("Unsupported platform");
180 }
181 }
182
183 /// Request CoIOMMU to unpin a specific range.
184 #[derive(Serialize, Deserialize, Debug)]
185 pub struct UnpinRequest {
186 /// The ranges presents (start gfn, count).
187 ranges: Vec<(u64, u64)>,
188 }
189
190 #[derive(Serialize, Deserialize, Debug)]
191 pub enum UnpinResponse {
192 Success,
193 Failed,
194 }
195
196 #[derive(Copy, Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
197 pub enum IommuDevType {
198 #[serde(rename = "off")]
199 #[default]
200 NoIommu,
201 #[serde(rename = "viommu")]
202 VirtioIommu,
203 #[serde(rename = "coiommu")]
204 CoIommu,
205 #[serde(rename = "pkvm-iommu")]
206 PkvmPviommu,
207 }
208
209 // Thread that handles commands sent to devices - such as snapshot, sleep, suspend
210 // Created when the VM is first created, and re-created on resumption of the VM.
create_devices_worker_thread( guest_memory: GuestMemory, io_bus: Arc<Bus>, mmio_bus: Arc<Bus>, device_ctrl_resp: Tube, ) -> std::io::Result<std::thread::JoinHandle<()>>211 pub fn create_devices_worker_thread(
212 guest_memory: GuestMemory,
213 io_bus: Arc<Bus>,
214 mmio_bus: Arc<Bus>,
215 device_ctrl_resp: Tube,
216 ) -> std::io::Result<std::thread::JoinHandle<()>> {
217 std::thread::Builder::new()
218 .name("device_control".to_string())
219 .spawn(move || {
220 let ex = Executor::new().expect("Failed to create an executor");
221
222 let async_control = AsyncTube::new(&ex, device_ctrl_resp).unwrap();
223 match ex.run_until(async move {
224 handle_command_tube(async_control, guest_memory, io_bus, mmio_bus).await
225 }) {
226 Ok(_) => {}
227 Err(e) => {
228 error!("Device control thread exited with error: {}", e);
229 }
230 };
231 })
232 }
233
sleep_buses(buses: &[&Bus]) -> anyhow::Result<()>234 fn sleep_buses(buses: &[&Bus]) -> anyhow::Result<()> {
235 for bus in buses {
236 bus.sleep_devices()
237 .with_context(|| format!("failed to sleep devices on {:?} bus", bus.get_bus_type()))?;
238 debug!("Devices slept successfully on {:?} bus", bus.get_bus_type());
239 }
240 Ok(())
241 }
242
wake_buses(buses: &[&Bus])243 fn wake_buses(buses: &[&Bus]) {
244 for bus in buses {
245 bus.wake_devices()
246 .with_context(|| format!("failed to wake devices on {:?} bus", bus.get_bus_type()))
247 // Some devices may have slept. Eternally.
248 // Recovery - impossible.
249 // Shut down VM.
250 .expect("VM panicked to avoid unexpected behavior");
251 debug!(
252 "Devices awoken successfully on {:?} Bus",
253 bus.get_bus_type()
254 );
255 }
256 }
257
258 // Use 64MB chunks when writing the memory snapshot (if encryption is used).
259 const MEMORY_SNAP_ENCRYPTED_CHUNK_SIZE_BYTES: usize = 1024 * 1024 * 64;
260
snapshot_handler( snapshot_writer: vm_control::SnapshotWriter, guest_memory: &GuestMemory, buses: &[&Bus], compress_memory: bool, ) -> anyhow::Result<()>261 async fn snapshot_handler(
262 snapshot_writer: vm_control::SnapshotWriter,
263 guest_memory: &GuestMemory,
264 buses: &[&Bus],
265 compress_memory: bool,
266 ) -> anyhow::Result<()> {
267 // SAFETY:
268 // VM & devices are stopped.
269 let guest_memory_metadata = unsafe {
270 guest_memory
271 .snapshot(
272 &mut snapshot_writer
273 .raw_fragment_with_chunk_size("mem", MEMORY_SNAP_ENCRYPTED_CHUNK_SIZE_BYTES)?,
274 compress_memory,
275 )
276 .context("failed to snapshot memory")?
277 };
278 snapshot_writer.write_fragment("mem_metadata", &guest_memory_metadata)?;
279 for (i, bus) in buses.iter().enumerate() {
280 bus.snapshot_devices(&snapshot_writer.add_namespace(&format!("bus{i}"))?)
281 .context("failed to snapshot bus devices")?;
282 debug!(
283 "Devices snapshot successfully for {:?} Bus",
284 bus.get_bus_type()
285 );
286 }
287 Ok(())
288 }
289
restore_handler( snapshot_reader: vm_control::SnapshotReader, guest_memory: &GuestMemory, buses: &[&Bus], ) -> anyhow::Result<()>290 async fn restore_handler(
291 snapshot_reader: vm_control::SnapshotReader,
292 guest_memory: &GuestMemory,
293 buses: &[&Bus],
294 ) -> anyhow::Result<()> {
295 let guest_memory_metadata = snapshot_reader.read_fragment("mem_metadata")?;
296 // SAFETY:
297 // VM & devices are stopped.
298 unsafe {
299 guest_memory.restore(
300 guest_memory_metadata,
301 &mut snapshot_reader.raw_fragment("mem")?,
302 )?
303 };
304 for (i, bus) in buses.iter().enumerate() {
305 bus.restore_devices(&snapshot_reader.namespace(&format!("bus{i}"))?)
306 .context("failed to restore bus devices")?;
307 debug!(
308 "Devices restore successfully for {:?} Bus",
309 bus.get_bus_type()
310 );
311 }
312 Ok(())
313 }
314
handle_command_tube( command_tube: AsyncTube, guest_memory: GuestMemory, io_bus: Arc<Bus>, mmio_bus: Arc<Bus>, ) -> anyhow::Result<()>315 async fn handle_command_tube(
316 command_tube: AsyncTube,
317 guest_memory: GuestMemory,
318 io_bus: Arc<Bus>,
319 mmio_bus: Arc<Bus>,
320 ) -> anyhow::Result<()> {
321 let buses = &[&*io_bus, &*mmio_bus];
322
323 // We assume devices are awake. This is safe because if the VM starts the
324 // sleeping state, run_control will ask us to sleep devices.
325 let mut devices_state = DevicesState::Wake;
326
327 loop {
328 match command_tube.next().await {
329 Ok(command) => {
330 match command {
331 DeviceControlCommand::SleepDevices => {
332 if let DevicesState::Wake = devices_state {
333 match sleep_buses(buses) {
334 Ok(()) => {
335 devices_state = DevicesState::Sleep;
336 }
337 Err(e) => {
338 error!("failed to sleep: {:#}", e);
339
340 // Failing to sleep could mean a single device failing to sleep.
341 // Wake up devices to resume functionality of the VM.
342 info!("Attempting to wake devices after failed sleep");
343 wake_buses(buses);
344
345 command_tube
346 .send(VmResponse::ErrString(e.to_string()))
347 .await
348 .context("failed to send response.")?;
349 continue;
350 }
351 }
352 }
353 command_tube
354 .send(VmResponse::Ok)
355 .await
356 .context("failed to reply to sleep command")?;
357 }
358 DeviceControlCommand::WakeDevices => {
359 if let DevicesState::Sleep = devices_state {
360 wake_buses(buses);
361 devices_state = DevicesState::Wake;
362 }
363 command_tube
364 .send(VmResponse::Ok)
365 .await
366 .context("failed to reply to wake devices request")?;
367 }
368 DeviceControlCommand::SnapshotDevices {
369 snapshot_writer,
370 compress_memory,
371 } => {
372 assert!(
373 matches!(devices_state, DevicesState::Sleep),
374 "devices must be sleeping to snapshot"
375 );
376 if let Err(e) =
377 snapshot_handler(snapshot_writer, &guest_memory, buses, compress_memory)
378 .await
379 {
380 error!("failed to snapshot: {:#}", e);
381 command_tube
382 .send(VmResponse::ErrString(e.to_string()))
383 .await
384 .context("Failed to send response")?;
385 continue;
386 }
387 command_tube
388 .send(VmResponse::Ok)
389 .await
390 .context("Failed to send response")?;
391 }
392 DeviceControlCommand::RestoreDevices { snapshot_reader } => {
393 assert!(
394 matches!(devices_state, DevicesState::Sleep),
395 "devices must be sleeping to restore"
396 );
397 if let Err(e) =
398 restore_handler(snapshot_reader, &guest_memory, &[&*io_bus, &*mmio_bus])
399 .await
400 {
401 error!("failed to restore: {:#}", e);
402 command_tube
403 .send(VmResponse::ErrString(e.to_string()))
404 .await
405 .context("Failed to send response")?;
406 continue;
407 }
408 command_tube
409 .send(VmResponse::Ok)
410 .await
411 .context("Failed to send response")?;
412 }
413 DeviceControlCommand::GetDevicesState => {
414 command_tube
415 .send(VmResponse::DevicesState(devices_state.clone()))
416 .await
417 .context("failed to send response")?;
418 }
419 DeviceControlCommand::Exit => {
420 return Ok(());
421 }
422 };
423 }
424 Err(e) => {
425 if matches!(e, TubeError::Disconnected) {
426 // Tube disconnected - shut down thread.
427 return Ok(());
428 }
429 return Err(anyhow!("Failed to receive: {}", e));
430 }
431 }
432 }
433 }
434