1 // Copyright 2022 The ChromiumOS Authors 2 // Use of this source code is governed by a BSD-style license that can be 3 // found in the LICENSE file. 4 5 use std::fs::File; 6 use std::sync::Arc; 7 8 use anyhow::bail; 9 use anyhow::Context; 10 use anyhow::Result; 11 use base::error; 12 use base::pagesize; 13 use base::AsRawDescriptor; 14 use base::AsRawDescriptors; 15 use base::Event; 16 use base::MappedRegion; 17 use base::MemoryMapping; 18 use base::MemoryMappingBuilder; 19 use base::Protection; 20 use base::RawDescriptor; 21 use hypervisor::MemCacheType; 22 use hypervisor::Vm; 23 use resources::SystemAllocator; 24 use vfio_sys::*; 25 use vm_control::api::VmMemoryClient; 26 use vm_control::VmMemoryDestination; 27 use vm_control::VmMemorySource; 28 use vm_memory::GuestAddress; 29 30 use crate::pci::CrosvmDeviceId; 31 use crate::vfio::VfioDevice; 32 use crate::vfio::VfioError; 33 use crate::vfio::VfioIrq; 34 use crate::BusAccessInfo; 35 use crate::BusDevice; 36 use crate::BusDeviceObj; 37 use crate::DeviceId; 38 use crate::IommuDevType; 39 use crate::IrqEdgeEvent; 40 use crate::IrqLevelEvent; 41 use crate::Suspendable; 42 43 struct MmioInfo { 44 index: usize, 45 start: u64, 46 length: u64, 47 } 48 49 pub struct VfioPlatformDevice { 50 device: Arc<VfioDevice>, 51 interrupt_edge_evt: Vec<IrqEdgeEvent>, 52 interrupt_level_evt: Vec<IrqLevelEvent>, 53 mmio_regions: Vec<MmioInfo>, 54 vm_memory_client: VmMemoryClient, 55 // scratch MemoryMapping to avoid unmap beform vm exit 56 mem: Vec<MemoryMapping>, 57 } 58 59 impl BusDevice for VfioPlatformDevice { device_id(&self) -> DeviceId60 fn device_id(&self) -> DeviceId { 61 CrosvmDeviceId::VfioPlatformDevice.into() 62 } 63 debug_label(&self) -> String64 fn debug_label(&self) -> String { 65 format!("vfio {} device", self.device.device_name()) 66 } 67 read(&mut self, info: BusAccessInfo, data: &mut [u8])68 fn read(&mut self, info: BusAccessInfo, data: &mut [u8]) { 69 self.read_mmio(info.address, data) 70 } 71 write(&mut self, info: BusAccessInfo, data: &[u8])72 fn write(&mut self, info: BusAccessInfo, data: &[u8]) { 73 self.write_mmio(info.address, data) 74 } 75 } 76 77 impl Suspendable for VfioPlatformDevice {} 78 79 impl BusDeviceObj for VfioPlatformDevice { as_platform_device(&self) -> Option<&VfioPlatformDevice>80 fn as_platform_device(&self) -> Option<&VfioPlatformDevice> { 81 Some(self) 82 } as_platform_device_mut(&mut self) -> Option<&mut VfioPlatformDevice>83 fn as_platform_device_mut(&mut self) -> Option<&mut VfioPlatformDevice> { 84 Some(self) 85 } into_platform_device(self: Box<Self>) -> Option<Box<VfioPlatformDevice>>86 fn into_platform_device(self: Box<Self>) -> Option<Box<VfioPlatformDevice>> { 87 Some(self) 88 } 89 } 90 91 impl VfioPlatformDevice { 92 /// Constructs a new Vfio Platform device for the given Vfio device new(device: VfioDevice, vm_memory_client: VmMemoryClient) -> Self93 pub fn new(device: VfioDevice, vm_memory_client: VmMemoryClient) -> Self { 94 let dev = Arc::new(device); 95 VfioPlatformDevice { 96 device: dev, 97 interrupt_edge_evt: Vec::new(), 98 interrupt_level_evt: Vec::new(), 99 mmio_regions: Vec::new(), 100 vm_memory_client, 101 mem: Vec::new(), 102 } 103 } 104 get_platform_irqs(&self) -> Result<Vec<VfioIrq>, VfioError>105 pub fn get_platform_irqs(&self) -> Result<Vec<VfioIrq>, VfioError> { 106 self.device.get_irqs() 107 } 108 irq_is_automask(&self, irq: &VfioIrq) -> bool109 pub fn irq_is_automask(&self, irq: &VfioIrq) -> bool { 110 irq.flags & VFIO_IRQ_INFO_AUTOMASKED != 0 111 } 112 setup_irq_resample(&mut self, resample_evt: &Event, index: u32) -> Result<()>113 fn setup_irq_resample(&mut self, resample_evt: &Event, index: u32) -> Result<()> { 114 self.device.irq_mask(index).context("Intx mask failed")?; 115 self.device 116 .resample_virq_enable(resample_evt, index) 117 .context("resample enable failed")?; 118 self.device 119 .irq_unmask(index) 120 .context("Intx unmask failed")?; 121 Ok(()) 122 } 123 assign_edge_platform_irq(&mut self, irq_evt: &IrqEdgeEvent, index: u32) -> Result<()>124 pub fn assign_edge_platform_irq(&mut self, irq_evt: &IrqEdgeEvent, index: u32) -> Result<()> { 125 let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?; 126 self.device 127 .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0) 128 .context("platform irq enable failed")?; 129 self.interrupt_edge_evt.push(interrupt_evt); 130 Ok(()) 131 } 132 assign_level_platform_irq(&mut self, irq_evt: &IrqLevelEvent, index: u32) -> Result<()>133 pub fn assign_level_platform_irq(&mut self, irq_evt: &IrqLevelEvent, index: u32) -> Result<()> { 134 let interrupt_evt = irq_evt.try_clone().context("failed to clone irq event")?; 135 self.device 136 .irq_enable(&[Some(interrupt_evt.get_trigger())], index, 0) 137 .context("platform irq enable failed")?; 138 if let Err(e) = self.setup_irq_resample(interrupt_evt.get_resample(), index) { 139 self.disable_irqs(index); 140 bail!("failed to set up irq resampling: {}", e); 141 } 142 self.interrupt_level_evt.push(interrupt_evt); 143 Ok(()) 144 } 145 find_region(&self, addr: u64) -> Option<MmioInfo>146 fn find_region(&self, addr: u64) -> Option<MmioInfo> { 147 for mmio_info in self.mmio_regions.iter() { 148 if addr >= mmio_info.start && addr < mmio_info.start + mmio_info.length { 149 return Some(MmioInfo { 150 index: mmio_info.index, 151 start: mmio_info.start, 152 length: mmio_info.length, 153 }); 154 } 155 } 156 None 157 } 158 allocate_regions( &mut self, resources: &mut SystemAllocator, ) -> Result<Vec<(u64, u64)>, resources::Error>159 pub fn allocate_regions( 160 &mut self, 161 resources: &mut SystemAllocator, 162 ) -> Result<Vec<(u64, u64)>, resources::Error> { 163 let mut ranges = Vec::new(); 164 for i in 0..self.device.get_region_count() { 165 let size = self.device.get_region_size(i); 166 let alloc_id = resources.get_anon_alloc(); 167 let allocator = resources 168 .mmio_platform_allocator() 169 .ok_or(resources::Error::MissingPlatformMMIOAddresses)?; 170 let start_addr = allocator.allocate_with_align( 171 size, 172 alloc_id, 173 "vfio_mmio".to_string(), 174 pagesize() as u64, 175 )?; 176 ranges.push((start_addr, size)); 177 178 self.mmio_regions.push(MmioInfo { 179 index: i, 180 start: start_addr, 181 length: size, 182 }); 183 } 184 Ok(ranges) 185 } 186 region_mmap_early(&self, vm: &mut impl Vm, index: usize, start_addr: u64)187 fn region_mmap_early(&self, vm: &mut impl Vm, index: usize, start_addr: u64) { 188 if self.device.get_region_flags(index) & VFIO_REGION_INFO_FLAG_MMAP == 0 { 189 return; 190 } 191 192 for mmap in &self.device.get_region_mmap(index) { 193 let mmap_offset = mmap.offset; 194 let mmap_size = mmap.size; 195 let guest_map_start = start_addr + mmap_offset; 196 let region_offset = self.device.get_region_offset(index); 197 let offset = region_offset + mmap_offset; 198 199 let mmap = match MemoryMappingBuilder::new(mmap_size as usize) 200 .from_file(self.device.device_file()) 201 .offset(offset) 202 .build() 203 { 204 Ok(v) => v, 205 Err(e) => { 206 error!("{e}, index: {index}, start_addr:{start_addr:#x}, offset:{offset:#x}"); 207 break; 208 } 209 }; 210 211 let host = mmap.as_ptr(); 212 let guest_addr = GuestAddress(guest_map_start); 213 if let Err(e) = vm.add_memory_region( 214 guest_addr, 215 Box::new(mmap), 216 false, 217 false, 218 MemCacheType::CacheCoherent, 219 ) { 220 error!("{e}, index: {index}, guest_addr:{guest_addr}, host:{host:?}"); 221 break; 222 } 223 } 224 } 225 226 /// Force adding the MMIO regions to the guest memory space. 227 /// 228 /// By default, MMIO regions are mapped lazily when the guest first accesses them. Instead, 229 /// this function maps them, even if the guest might end up not accessing them. It only runs in 230 /// the current thread and can therefore be called before the VM is started. regions_mmap_early(&mut self, vm: &mut impl Vm)231 pub fn regions_mmap_early(&mut self, vm: &mut impl Vm) { 232 for mmio_info in self.mmio_regions.iter() { 233 self.region_mmap_early(vm, mmio_info.index, mmio_info.start); 234 } 235 } 236 region_mmap(&self, index: usize, start_addr: u64) -> Vec<MemoryMapping>237 fn region_mmap(&self, index: usize, start_addr: u64) -> Vec<MemoryMapping> { 238 let mut mem_map: Vec<MemoryMapping> = Vec::new(); 239 if self.device.get_region_flags(index) & VFIO_REGION_INFO_FLAG_MMAP != 0 { 240 let mmaps = self.device.get_region_mmap(index); 241 if mmaps.is_empty() { 242 return mem_map; 243 } 244 245 for mmap in mmaps.iter() { 246 let mmap_offset = mmap.offset; 247 let mmap_size = mmap.size; 248 let guest_map_start = start_addr + mmap_offset; 249 let region_offset = self.device.get_region_offset(index); 250 let offset = region_offset + mmap_offset; 251 let descriptor = match self.device.device_file().try_clone() { 252 Ok(device_file) => device_file.into(), 253 Err(_) => break, 254 }; 255 match self.vm_memory_client.register_memory( 256 VmMemorySource::Descriptor { 257 descriptor, 258 offset, 259 size: mmap_size, 260 }, 261 VmMemoryDestination::GuestPhysicalAddress(guest_map_start), 262 Protection::read_write(), 263 MemCacheType::CacheCoherent, 264 ) { 265 Ok(_region) => { 266 // Even if vm has mapped this region, but it is in vm main process, 267 // device process doesn't has this mapping, but vfio_dma_map() need it 268 // in device process, so here map it again. 269 let mmap = match MemoryMappingBuilder::new(mmap_size as usize) 270 .from_file(self.device.device_file()) 271 .offset(offset) 272 .build() 273 { 274 Ok(v) => v, 275 Err(_e) => break, 276 }; 277 let host = mmap.as_ptr() as u64; 278 // SAFETY: 279 // Safe because the given guest_map_start is valid guest bar address. and 280 // the host pointer is correct and valid guaranteed by MemoryMapping 281 // interface. 282 match unsafe { 283 self.device 284 .vfio_dma_map(guest_map_start, mmap_size, host, true) 285 } { 286 Ok(_) => mem_map.push(mmap), 287 Err(e) => { 288 error!( 289 "{}, index: {}, start_addr:0x{:x}, host:0x{:x}", 290 e, index, start_addr, host 291 ); 292 break; 293 } 294 } 295 } 296 Err(e) => { 297 error!("register_memory failed: {}", e); 298 break; 299 } 300 } 301 } 302 } 303 304 mem_map 305 } 306 regions_mmap(&mut self)307 fn regions_mmap(&mut self) { 308 for mmio_info in self.mmio_regions.iter() { 309 let mut mem_map = self.region_mmap(mmio_info.index, mmio_info.start); 310 self.mem.append(&mut mem_map); 311 } 312 } 313 disable_irqs(&mut self, index: u32)314 fn disable_irqs(&mut self, index: u32) { 315 if let Err(e) = self.device.irq_disable(index) { 316 error!("Platform irq disable failed: {}", e); 317 } 318 } 319 read_mmio(&mut self, addr: u64, data: &mut [u8])320 fn read_mmio(&mut self, addr: u64, data: &mut [u8]) { 321 if let Some(mmio_info) = self.find_region(addr) { 322 let offset = addr - mmio_info.start; 323 let index = mmio_info.index; 324 self.device.region_read(index, data, offset); 325 } 326 // We have no other way than wait for 1st access and then do the mmap, 327 // so that next accesses are dual-stage MMU accelerated. 328 self.regions_mmap(); 329 } 330 write_mmio(&mut self, addr: u64, data: &[u8])331 fn write_mmio(&mut self, addr: u64, data: &[u8]) { 332 if let Some(mmio_info) = self.find_region(addr) { 333 let offset = addr - mmio_info.start; 334 let index = mmio_info.index; 335 self.device.region_write(index, data, offset); 336 } 337 // We have no other way than wait for 1st access and then do the mmap, 338 // so that next accesses are dual-stage MMU accelerated. 339 self.regions_mmap(); 340 } 341 keep_rds(&self) -> Vec<RawDescriptor>342 pub fn keep_rds(&self) -> Vec<RawDescriptor> { 343 let mut rds = self.device.keep_rds(); 344 345 for irq_evt in self.interrupt_edge_evt.iter() { 346 rds.extend(irq_evt.as_raw_descriptors()); 347 } 348 349 for irq_evt in self.interrupt_level_evt.iter() { 350 rds.extend(irq_evt.as_raw_descriptors()); 351 } 352 353 rds.push(self.vm_memory_client.as_raw_descriptor()); 354 rds 355 } 356 357 /// Gets the vfio device backing `File`. device_file(&self) -> &File358 pub fn device_file(&self) -> &File { 359 self.device.device_file() 360 } 361 362 /// Returns the DT symbol (node label) of the VFIO device. dt_symbol(&self) -> Option<&str>363 pub fn dt_symbol(&self) -> Option<&str> { 364 self.device.dt_symbol() 365 } 366 367 /// Returns the type and indentifier (if applicable) of the IOMMU used by this VFIO device and 368 /// its master IDs. iommu(&self) -> Option<(IommuDevType, Option<u32>, &[u32])>369 pub fn iommu(&self) -> Option<(IommuDevType, Option<u32>, &[u32])> { 370 self.device.iommu() 371 } 372 } 373