1 // Copyright 2024 The ChromiumOS Authors
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4
5 #![cfg(target_arch = "x86_64")]
6 #![cfg(any(feature = "whpx", feature = "gvm", feature = "haxm", unix))]
7
8 use core::mem;
9 use std::arch::asm;
10 use std::cell::RefCell;
11 use std::ffi::c_void;
12 use std::sync::atomic::AtomicU8;
13 use std::sync::atomic::Ordering;
14 use std::sync::Arc;
15
16 use base::set_cpu_affinity;
17 use base::MappedRegion;
18 use base::MemoryMappingBuilder;
19 use base::SharedMemory;
20 #[cfg(feature = "gvm")]
21 use hypervisor::gvm::*;
22 #[cfg(all(windows, feature = "haxm"))]
23 use hypervisor::haxm::*;
24 #[cfg(any(target_os = "android", target_os = "linux"))]
25 use hypervisor::kvm::*;
26 #[cfg(all(windows, feature = "whpx"))]
27 use hypervisor::whpx::*;
28 #[cfg(any(target_os = "android", target_os = "linux"))]
29 use hypervisor::MemCacheType::CacheCoherent;
30 use hypervisor::*;
31 use hypervisor_test_macro::global_asm_data;
32 use sync::Mutex;
33 use vm_memory::GuestAddress;
34 use vm_memory::GuestMemory;
35 #[cfg(windows)]
36 use windows::Win32::System::Memory::VirtualLock;
37 #[cfg(windows)]
38 use windows::Win32::System::Memory::VirtualUnlock;
39 use zerocopy::AsBytes;
40
41 const FLAGS_IF_BIT: u64 = 0x200;
42
43 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
44 pub enum HypervisorType {
45 Kvm,
46 Whpx,
47 Haxm,
48 Gvm,
49 }
50
51 #[repr(C, packed)]
52 #[derive(AsBytes)]
53 /// Define IDTR value used in real mode or 32bit protected mode.
54 struct Idtr32 {
55 // The lower 2 bytes are limit.
56 limit: u16,
57 // The higher 4 bytes are base address.
58 base_address: u32,
59 }
60
61 #[repr(C, packed)]
62 #[derive(AsBytes, Debug, Copy, Clone)]
63 /// IDT entries for long mode.
64 struct IdtEntry64 {
65 address_low: u16,
66 selector: u16,
67 ist: u8,
68 flags: u8,
69 address_mid: u16,
70 address_high: u32,
71 reserved: u32,
72 }
73
74 impl IdtEntry64 {
new(handler_addr: u64) -> Self75 pub fn new(handler_addr: u64) -> Self {
76 IdtEntry64 {
77 address_low: (handler_addr & 0xFFFF) as u16,
78 selector: 0x10, // Our long mode CS is the third entry (0x0, 0x8, 0x10).
79 ist: 0,
80 flags: 0x8E, // Present, interrupt gate, DPL 0
81 address_mid: ((handler_addr >> 16) & 0xFFFF) as u16,
82 address_high: (handler_addr >> 32) as u32,
83 reserved: 0,
84 }
85 }
86 }
87
88 impl std::fmt::Display for HypervisorType {
fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result89 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
90 match self {
91 HypervisorType::Kvm => write!(f, "KVM"),
92 HypervisorType::Whpx => write!(f, "WHPX"),
93 HypervisorType::Haxm => write!(f, "HAXM"),
94 HypervisorType::Gvm => write!(f, "GVM"),
95 }
96 }
97 }
98
99 pub trait HypervisorTestSetup {
100 type Hypervisor: Hypervisor;
101 type Vm: VmX86_64;
102
create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm)103 fn create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm);
104 }
105
106 #[cfg(any(target_os = "android", target_os = "linux"))]
107 impl HypervisorTestSetup for Kvm {
108 type Hypervisor = Kvm;
109 type Vm = KvmVm;
110
create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm)111 fn create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm) {
112 let kvm = Kvm::new().expect("failed to create kvm");
113 let vm = KvmVm::new(&kvm, guest_mem, Default::default()).expect("failed to create vm");
114 (kvm, vm)
115 }
116 }
117
118 #[cfg(all(windows, feature = "whpx"))]
119 impl HypervisorTestSetup for Whpx {
120 type Hypervisor = Whpx;
121 type Vm = WhpxVm;
122
create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm)123 fn create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm) {
124 let whpx = Whpx::new().expect("failed to create whpx");
125 let vm = WhpxVm::new(&whpx, 1, guest_mem, CpuId::new(0), false, None)
126 .expect("failed to create vm");
127 (whpx, vm)
128 }
129 }
130
131 #[cfg(all(windows, feature = "haxm"))]
132 impl HypervisorTestSetup for Haxm {
133 type Hypervisor = Haxm;
134 type Vm = HaxmVm;
135
create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm)136 fn create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm) {
137 let haxm = Haxm::new().expect("failed to create haxm");
138 let vm = HaxmVm::new(&haxm, guest_mem).expect("failed to create vm");
139 (haxm, vm)
140 }
141 }
142
143 #[cfg(feature = "gvm")]
144 impl HypervisorTestSetup for Gvm {
145 type Hypervisor = Gvm;
146 type Vm = GvmVm;
147
create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm)148 fn create_vm(guest_mem: GuestMemory) -> (Self::Hypervisor, Self::Vm) {
149 let gvm = Gvm::new().expect("failed to create gvm");
150 let vm = GvmVm::new(&gvm, guest_mem).expect("failed to create vm");
151 (gvm, vm)
152 }
153 }
154
155 pub struct TestSetup {
156 pub assembly: Vec<u8>,
157 pub load_addr: GuestAddress,
158 pub mem_size: u64,
159 pub initial_regs: Regs,
160 pub extra_vm_setup: Option<Box<dyn Fn(&mut dyn VcpuX86_64, &mut dyn Vm) + Send>>,
161 pub memory_initializations: Vec<(GuestAddress, Vec<u8>)>,
162 pub expect_run_success: bool,
163
164 /// Whether the `exit_matcher` should recieve [`VcpuExit::Intr`]. Default to `false`.
165 ///
166 /// Hypervisors may occasinally receive [`VcpuExit::Intr`] if external interrupt intercept is
167 /// enabled. In such case, we should proceed to the next VCPU run to handle it. HAXM doesn't
168 /// distinguish between [`VcpuExit::Intr`] and [`VcpuExit::IrqWindowOpen`], so it may be
169 /// necessary to intercept [`VcpuExit::Intr`] for testing
170 /// [`VcpuX86_64::set_interrupt_window_requested`].
171 pub intercept_intr: bool,
172 }
173
174 impl Default for TestSetup {
default() -> Self175 fn default() -> Self {
176 TestSetup {
177 assembly: Vec::new(),
178 load_addr: GuestAddress(0),
179 mem_size: 0xF000, // Big enough default for long mode setup
180 initial_regs: Regs::default(),
181 extra_vm_setup: None,
182 memory_initializations: Vec::new(),
183 expect_run_success: true,
184 intercept_intr: false,
185 }
186 }
187 }
188
189 impl TestSetup {
new() -> Self190 pub fn new() -> Self {
191 Default::default()
192 }
193
add_memory_initialization(&mut self, addr: GuestAddress, data: Vec<u8>)194 pub fn add_memory_initialization(&mut self, addr: GuestAddress, data: Vec<u8>) {
195 self.memory_initializations.push((addr, data));
196 }
197 }
198
run_configurable_test<H: HypervisorTestSetup>( hypervisor_type: HypervisorType, setup: &TestSetup, regs_matcher: impl Fn(HypervisorType, &Regs, &Sregs), mut exit_matcher: impl FnMut(HypervisorType, &VcpuExit, &mut dyn VcpuX86_64, &mut dyn Vm) -> bool, )199 pub fn run_configurable_test<H: HypervisorTestSetup>(
200 hypervisor_type: HypervisorType,
201 setup: &TestSetup,
202 regs_matcher: impl Fn(HypervisorType, &Regs, &Sregs),
203 mut exit_matcher: impl FnMut(HypervisorType, &VcpuExit, &mut dyn VcpuX86_64, &mut dyn Vm) -> bool,
204 ) {
205 println!("Running test on hypervisor: {}", hypervisor_type);
206
207 let guest_mem =
208 GuestMemory::new(&[(GuestAddress(0), setup.mem_size)]).expect("failed to create guest mem");
209
210 for (addr, data) in &setup.memory_initializations {
211 guest_mem
212 .write_at_addr(data, *addr)
213 .expect("failed to write memory initialization");
214 }
215
216 guest_mem
217 .write_at_addr(&setup.assembly, setup.load_addr)
218 .expect("failed to write to guest memory");
219
220 let (_, mut vm) = H::create_vm(guest_mem);
221
222 let mut vcpu = vm.create_vcpu(0).expect("new vcpu failed");
223
224 let mut sregs = vcpu.get_sregs().expect("get sregs failed");
225 sregs.cs.base = 0;
226 sregs.cs.selector = 0;
227 vcpu.set_sregs(&sregs).expect("set sregs failed");
228 vcpu.set_regs(&setup.initial_regs).expect("set regs failed");
229
230 if let Some(ref setup_fn) = setup.extra_vm_setup {
231 setup_fn(&mut *vcpu, &mut vm);
232 }
233
234 if !vm.check_capability(VmCap::EarlyInitCpuid) {
235 let cpuid = vm
236 .get_hypervisor()
237 .get_supported_cpuid()
238 .expect("get_supported_cpuid() failed");
239 vcpu.set_cpuid(&cpuid).expect("set_cpuid() failed");
240 }
241
242 loop {
243 match vcpu.run() {
244 Ok(exit) => match exit {
245 // Handle interrupts by continuing the loop
246 VcpuExit::Intr if !setup.intercept_intr => continue,
247 other_exit => {
248 if !setup.expect_run_success {
249 panic!("Expected vcpu.run() to fail, but it succeeded");
250 }
251 if exit_matcher(hypervisor_type, &other_exit, &mut *vcpu, &mut vm) {
252 break;
253 }
254 }
255 },
256 Err(e) => {
257 if setup.expect_run_success {
258 panic!(
259 "Expected vcpu.run() to succeed, but it failed with error: {:?}",
260 e
261 );
262 } else {
263 println!("Expected failure occurred: {:?}", e);
264 break;
265 }
266 }
267 }
268 }
269
270 let final_regs = vcpu.get_regs().expect("failed to get regs");
271 let final_sregs = vcpu.get_sregs().expect("failed to get sregs");
272
273 regs_matcher(hypervisor_type, &final_regs, &final_sregs);
274 }
275
276 macro_rules! run_tests {
277 ($setup:expr, $regs_matcher:expr, $exit_matcher:expr) => {
278 #[cfg(any(target_os = "android", target_os = "linux"))]
279 run_configurable_test::<Kvm>(HypervisorType::Kvm, &$setup, $regs_matcher, $exit_matcher);
280
281 #[cfg(all(windows, feature = "whpx"))]
282 run_configurable_test::<Whpx>(HypervisorType::Whpx, &$setup, $regs_matcher, $exit_matcher);
283
284 #[cfg(all(windows, feature = "haxm"))]
285 run_configurable_test::<Haxm>(HypervisorType::Haxm, &$setup, $regs_matcher, $exit_matcher);
286
287 #[cfg(feature = "gvm")]
288 run_configurable_test::<Gvm>(HypervisorType::Gvm, &$setup, $regs_matcher, $exit_matcher);
289 };
290 }
291
292 const DEFAULT_GDT_OFFSET: u64 = 0x1500;
293 const DEFAULT_IDT_OFFSET: u64 = 0x1528;
294
295 const DESC_ACCESS_EXEC: u8 = 1 << 3;
296 const DESC_ACCESS_RW: u8 = 1 << 1;
297 const DESC_ACCESS_ACCESSED: u8 = 1 << 0;
298
299 #[derive(Debug, Clone, Copy)]
300 struct LongModePageTableEntry {
301 execute_disable: bool,
302 protection_key: u8,
303 address: u64,
304 global: bool,
305 page_attribute_table: bool,
306 dirty: bool,
307 accessed: bool,
308 cache_disable: bool,
309 write_through: bool,
310 user_supervisor: bool,
311 read_write: bool,
312 present: bool,
313 }
314
315 impl LongModePageTableEntry {
from_address(address: u64) -> Self316 fn from_address(address: u64) -> Self {
317 assert!(address < 1 << 52, "the address must fit in 52 bits");
318 assert!(address & 0xFFF == 0, "the address must be aligned to 4k");
319 Self {
320 execute_disable: false,
321 protection_key: 0,
322 address,
323 global: false,
324 page_attribute_table: false,
325 dirty: false,
326 accessed: false,
327 cache_disable: false,
328 write_through: false,
329 user_supervisor: false,
330 read_write: false,
331 present: false,
332 }
333 }
334 }
335
336 impl From<LongModePageTableEntry> for u64 {
from(page_table_entry: LongModePageTableEntry) -> Self337 fn from(page_table_entry: LongModePageTableEntry) -> Self {
338 let mut res = 0;
339 if page_table_entry.present {
340 res |= 1;
341 }
342 if page_table_entry.read_write {
343 res |= 1 << 1;
344 }
345 if page_table_entry.user_supervisor {
346 res |= 1 << 2;
347 }
348 if page_table_entry.write_through {
349 res |= 1 << 3;
350 }
351 if page_table_entry.cache_disable {
352 res |= 1 << 4;
353 }
354 if page_table_entry.accessed {
355 res |= 1 << 5;
356 }
357 if page_table_entry.dirty {
358 res |= 1 << 6;
359 }
360 if page_table_entry.page_attribute_table {
361 res |= 1 << 7;
362 }
363 if page_table_entry.global {
364 res |= 1 << 8;
365 }
366 assert!(page_table_entry.address < 1 << 52);
367 assert!(page_table_entry.address & 0xFFF == 0);
368 res |= page_table_entry.address;
369 assert!(page_table_entry.protection_key < 1 << 4);
370 res |= u64::from(page_table_entry.protection_key) << 59;
371 if page_table_entry.execute_disable {
372 res |= 1 << 63;
373 }
374 res
375 }
376 }
377
378 #[derive(Debug, Clone)]
379 struct ModeConfig {
380 idt: Vec<u8>,
381 idt_base_addr: u64,
382 gdt: Vec<Segment>,
383 gdt_base_addr: u64,
384 code_segment_index: u16,
385 task_segment_index: Option<u16>,
386 page_table: Option<Box<[u8; 0x1000]>>,
387 long_mode: bool,
388 }
389
390 impl ModeConfig {
391 const IDT64_SIZE: usize = std::mem::size_of::<IdtEntry64>() * 256;
392 const IDT32_SIZE: usize = 8 * 256;
393
394 /// Set the IDT for long mode.
set_idt_long_mode(&mut self, idt: impl IntoIterator<Item = IdtEntry64>) -> &mut Self395 fn set_idt_long_mode(&mut self, idt: impl IntoIterator<Item = IdtEntry64>) -> &mut Self {
396 let entries = idt.into_iter().collect::<Vec<_>>();
397 assert_eq!(entries.len(), 256, "IDT must contain 256 entries");
398 self.idt = entries
399 .into_iter()
400 .flat_map(|entry| entry.as_bytes().to_owned())
401 .collect();
402 self
403 }
404
set_idt_base_addr(&mut self, idt_base_addr: u64) -> &mut Self405 fn set_idt_base_addr(&mut self, idt_base_addr: u64) -> &mut Self {
406 self.idt_base_addr = idt_base_addr;
407 self
408 }
409
default_code_segment_long_mode() -> Segment410 fn default_code_segment_long_mode() -> Segment {
411 Segment {
412 base: 0,
413 limit_bytes: 0xffff_ffff,
414 type_: DESC_ACCESS_EXEC | DESC_ACCESS_RW | DESC_ACCESS_ACCESSED,
415 present: 1,
416 dpl: 0,
417 db: 0,
418 s: 1,
419 l: 1,
420 g: 1,
421 ..Default::default()
422 }
423 }
424
default_code_segment_protected_mode() -> Segment425 fn default_code_segment_protected_mode() -> Segment {
426 Segment {
427 base: 0,
428 limit_bytes: 0xffff_ffff,
429 type_: DESC_ACCESS_EXEC | DESC_ACCESS_RW | DESC_ACCESS_ACCESSED,
430 present: 1,
431 dpl: 0,
432 db: 1,
433 s: 1,
434 l: 0,
435 g: 1,
436 ..Default::default()
437 }
438 }
439
segment_to_bytes(segment: &Segment, long_mode: bool) -> Vec<u8>440 fn segment_to_bytes(segment: &Segment, long_mode: bool) -> Vec<u8> {
441 if *segment == Segment::default() {
442 // Special handle for null descriptor, so that it won't be recognized as a 64
443 // bit system segment.
444 return vec![0u8; 8];
445 }
446 let Segment {
447 base,
448 limit_bytes,
449 type_,
450 present,
451 dpl,
452 db,
453 s,
454 l,
455 g,
456 ..
457 } = *segment;
458
459 let limit = if g != 0 {
460 // 4096-byte granularity
461 limit_bytes / 4096
462 } else {
463 // 1-byte granularity
464 limit_bytes
465 };
466
467 assert!(limit < (1 << 20)); // limit value must fit in 20 bits
468 let flags = {
469 let mut flags = 0;
470 if g != 0 {
471 flags |= 1 << 3;
472 }
473 if db != 0 {
474 flags |= 1 << 2;
475 }
476 if l != 0 {
477 flags |= 1 << 1;
478 }
479 flags << 4
480 };
481 assert!(flags & 0x0F == 0x00); // flags must be in the high 4 bits only
482 let access = {
483 assert!(type_ < (1 << 4), "type must fit in 4 bits");
484 let mut access = type_;
485 if present != 0 {
486 access |= 1 << 7;
487 }
488 assert!(dpl < (1 << 2), "DPL must fit in 2 bits");
489 access |= dpl << 5;
490 if s != 0 {
491 access |= 1 << 4;
492 }
493 access
494 };
495
496 let limit_lo = (limit & 0xffff).try_into().unwrap();
497 let base_lo = (base & 0xffff).try_into().unwrap();
498 let base_mid0 = ((base >> 16) & 0xff).try_into().unwrap();
499 let limit_hi_and_flags = u8::try_from((limit >> 16) & 0xf).unwrap() | flags;
500 let base_mid1 = ((base >> 24) & 0xff).try_into().unwrap();
501 let base_hi = (base >> 32).try_into().unwrap();
502
503 if long_mode && s == 0 {
504 // 64 bit system segment descriptor.
505 #[repr(C, packed)]
506 #[derive(AsBytes)]
507 struct Descriptor {
508 limit_lo: u16,
509 base_lo: u16,
510 base_mid0: u8,
511 access: u8,
512 limit_hi_and_flags: u8,
513 base_mid1: u8,
514 base_hi: u32,
515 _reserved: [u8; 4],
516 }
517
518 Descriptor {
519 limit_lo,
520 base_lo,
521 base_mid0,
522 access,
523 limit_hi_and_flags,
524 base_mid1,
525 base_hi,
526 _reserved: [0; 4],
527 }
528 .as_bytes()
529 .to_owned()
530 } else {
531 #[repr(C, packed)]
532 #[derive(AsBytes)]
533 struct Descriptor {
534 limit_lo: u16,
535 base_lo: u16,
536 base_mid: u8,
537 access: u8,
538 limit_hi_and_flags: u8,
539 base_hi: u8,
540 }
541
542 assert_eq!(base_hi, 0, "the base address must be within 32 bit range");
543 Descriptor {
544 limit_lo,
545 base_lo,
546 base_mid: base_mid0,
547 access,
548 limit_hi_and_flags,
549 base_hi: base_mid1,
550 }
551 .as_bytes()
552 .to_owned()
553 }
554 }
555
get_gdt_bytes(&self) -> Vec<u8>556 fn get_gdt_bytes(&self) -> Vec<u8> {
557 self.gdt
558 .iter()
559 .flat_map(|segment| Self::segment_to_bytes(segment, self.long_mode))
560 .collect()
561 }
562
configure_gdt_memory(&self, guest_mem: &GuestMemory)563 fn configure_gdt_memory(&self, guest_mem: &GuestMemory) {
564 let gdt_bytes = self.get_gdt_bytes();
565 let gdt_start_addr = GuestAddress(self.gdt_base_addr);
566 let gdt_end_addr = gdt_start_addr
567 .checked_add(
568 gdt_bytes
569 .len()
570 .try_into()
571 .expect("the GDT size must be within usize"),
572 )
573 .expect("the end of GDT address shouldn't overflow");
574 assert!(
575 guest_mem.range_overlap(GuestAddress(self.gdt_base_addr), gdt_end_addr),
576 "the address for GDT is not mapped"
577 );
578 guest_mem
579 .write_at_addr(&gdt_bytes, GuestAddress(self.gdt_base_addr))
580 .expect("Failed to write GDT entry to guest memory");
581 }
582
configure_idt_memory(&self, guest_mem: &GuestMemory)583 fn configure_idt_memory(&self, guest_mem: &GuestMemory) {
584 let expected_length = if self.long_mode {
585 Self::IDT64_SIZE
586 } else {
587 Self::IDT32_SIZE
588 };
589
590 let idt_addr = GuestAddress(self.idt_base_addr);
591 assert_eq!(self.idt.len(), expected_length);
592 assert!(
593 guest_mem.range_overlap(
594 idt_addr,
595 idt_addr
596 .checked_add(
597 self.idt
598 .len()
599 .try_into()
600 .expect("The IDT length must be within the u64 range.")
601 )
602 .expect("The end address of IDT should not overflow")
603 ),
604 "The IDT that starts at {:#x} isn't properly mapped as the guest memory.",
605 self.idt_base_addr
606 );
607 guest_mem
608 .write_at_addr(&self.idt, idt_addr)
609 .expect("failed to write IDT entry to guest memory");
610 }
611
get_idtr_value(&self) -> DescriptorTable612 fn get_idtr_value(&self) -> DescriptorTable {
613 DescriptorTable {
614 base: self.idt_base_addr,
615 limit: {
616 let expected_length = if self.long_mode {
617 Self::IDT64_SIZE
618 } else {
619 Self::IDT32_SIZE
620 };
621 assert_eq!(self.idt.len(), expected_length, "the IDT size should match",);
622 // The IDT limit should be the number of bytes of IDT - 1.
623 (self.idt.len() - 1)
624 .try_into()
625 .expect("the IDT limit should be within the range of u16")
626 },
627 }
628 }
629
get_gdtr_value(&self) -> DescriptorTable630 fn get_gdtr_value(&self) -> DescriptorTable {
631 DescriptorTable {
632 base: self.gdt_base_addr,
633 limit: (self.get_gdt_bytes().len() - 1)
634 .try_into()
635 .expect("the GDT limit should fit in 16 bits"),
636 }
637 }
638
get_segment_register_value(&self, segment_index: u16) -> Segment639 fn get_segment_register_value(&self, segment_index: u16) -> Segment {
640 let offset: usize = self
641 .gdt
642 .iter()
643 .take(segment_index.into())
644 .map(|segment| Self::segment_to_bytes(segment, self.long_mode).len())
645 .sum();
646 Segment {
647 selector: offset
648 .try_into()
649 .expect("the offset should be within the range of u16"),
650 ..self.gdt[usize::from(segment_index)]
651 }
652 }
653
configure_long_mode_memory(&self, vm: &mut dyn Vm)654 pub fn configure_long_mode_memory(&self, vm: &mut dyn Vm) {
655 let guest_mem = vm.get_memory();
656
657 self.configure_gdt_memory(guest_mem);
658 self.configure_idt_memory(guest_mem);
659
660 // Setup paging
661 let pml4_addr = GuestAddress(0x9000);
662 let pdpte_addr = GuestAddress(0xa000);
663 let pde_addr = GuestAddress(0xb000);
664 let pte_addr = GuestAddress(0xc000);
665
666 assert!(
667 guest_mem.range_overlap(GuestAddress(0x9000), GuestAddress(0xd000)),
668 "the memory range for page tables should be mapped."
669 );
670
671 // Pointing to PDPTE with present and RW flags
672 guest_mem
673 .write_at_addr(&(pdpte_addr.0 | 3).to_le_bytes(), pml4_addr)
674 .expect("failed to write PML4 entry");
675
676 // Pointing to PD with present and RW flags
677 guest_mem
678 .write_at_addr(&(pde_addr.0 | 3).to_le_bytes(), pdpte_addr)
679 .expect("failed to write PDPTE entry");
680
681 for i in 0..512 {
682 // All pages are present and RW.
683 let flags: u64 = if i == 0 {
684 3
685 } else {
686 // The first 2MiB are 4K pages, the rest are 2M pages.
687 0x83
688 };
689 let addr = if i == 0 { pte_addr.offset() } else { i << 21 };
690 let pd_entry_bytes = (addr | flags).to_le_bytes();
691 guest_mem
692 .write_at_addr(
693 &pd_entry_bytes,
694 pde_addr.unchecked_add(i * mem::size_of::<u64>() as u64),
695 )
696 .expect("Failed to write PDE entry");
697 }
698
699 guest_mem
700 .write_at_addr(
701 self.page_table
702 .as_ref()
703 .expect("page table must present for long mode")
704 .as_slice(),
705 pte_addr,
706 )
707 .expect("Failed to write PTE entry");
708 }
709
enter_long_mode(&self, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm)710 pub fn enter_long_mode(&self, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm) {
711 self.configure_long_mode_memory(vm);
712
713 let mut sregs = vcpu.get_sregs().expect("failed to get sregs");
714
715 sregs.gdt = self.get_gdtr_value();
716 sregs.idt = self.get_idtr_value();
717 sregs.cs = self.get_segment_register_value(self.code_segment_index);
718
719 if let Some(task_segment_index) = self.task_segment_index {
720 sregs.tr = self.get_segment_register_value(task_segment_index);
721 }
722
723 // Long mode
724 let pml4_addr = GuestAddress(0x9000);
725 sregs.cr0 |= 0x1 | 0x80000000; // PE & PG
726 sregs.efer |= 0x100 | 0x400; // LME & LMA (Must be auto-enabled with CR0_PG)
727 sregs.cr3 = pml4_addr.offset();
728 sregs.cr4 |= 0x80 | 0x20; // PGE & PAE
729
730 vcpu.set_sregs(&sregs).expect("failed to set sregs");
731 }
732
configure_flat_protected_mode_memory(&self, vm: &mut dyn Vm)733 pub fn configure_flat_protected_mode_memory(&self, vm: &mut dyn Vm) {
734 let guest_mem = vm.get_memory();
735
736 self.configure_gdt_memory(guest_mem);
737 self.configure_idt_memory(guest_mem);
738 }
739
enter_protected_mode(&self, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm)740 pub fn enter_protected_mode(&self, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm) {
741 self.configure_flat_protected_mode_memory(vm);
742
743 let mut sregs = vcpu.get_sregs().expect("failed to get sregs");
744
745 sregs.cs = self.get_segment_register_value(self.code_segment_index);
746 sregs.gdt = self.get_gdtr_value();
747 sregs.idt = self.get_idtr_value();
748
749 assert!(
750 self.task_segment_index.is_none(),
751 "task segment not supported for protected mode yet."
752 );
753
754 assert!(
755 self.page_table.is_none(),
756 "setting page tables for protected mode is not supported yet"
757 );
758 // 32-bit protected mode, paging disabled
759 sregs.cr0 |= 0x1; // PE
760 sregs.cr0 &= !0x80000000; // ~PG
761
762 vcpu.set_sregs(&sregs).expect("failed to set sregs");
763 }
764
default_long_mode() -> Self765 fn default_long_mode() -> Self {
766 let page_table = (0u64..512)
767 .flat_map(|page_frame_number| {
768 let page_table_entry = LongModePageTableEntry {
769 present: true,
770 read_write: true,
771 ..LongModePageTableEntry::from_address(page_frame_number << 12)
772 };
773 u64::from(page_table_entry).as_bytes().to_owned()
774 })
775 .collect::<Box<[u8]>>()
776 .try_into()
777 .expect("the length of the slice must match");
778 Self {
779 idt_base_addr: DEFAULT_IDT_OFFSET,
780 idt: vec![0; Self::IDT64_SIZE],
781 gdt_base_addr: DEFAULT_GDT_OFFSET,
782 gdt: vec![
783 Segment::default(),
784 Segment::default(),
785 Self::default_code_segment_long_mode(),
786 ],
787 code_segment_index: 2,
788 task_segment_index: None,
789 page_table: Some(page_table),
790 long_mode: true,
791 }
792 }
793
default_protected_mode() -> Self794 fn default_protected_mode() -> Self {
795 Self {
796 idt_base_addr: DEFAULT_IDT_OFFSET,
797 idt: vec![0; Self::IDT32_SIZE],
798 gdt_base_addr: DEFAULT_GDT_OFFSET,
799 gdt: vec![
800 Segment::default(),
801 Segment::default(),
802 Self::default_code_segment_protected_mode(),
803 ],
804 code_segment_index: 2,
805 task_segment_index: None,
806 page_table: None,
807 long_mode: false,
808 }
809 }
810 }
811
812 global_asm_data!(
813 test_minimal_virtualization_code,
814 ".code16",
815 "add ax, bx",
816 "hlt"
817 );
818
819 // This runs a minimal program under virtualization.
820 // It should require only the ability to execute instructions under virtualization, physical
821 // memory, the ability to get and set some guest VM registers, and intercepting HLT.
822 #[test]
test_minimal_virtualization()823 fn test_minimal_virtualization() {
824 let assembly = test_minimal_virtualization_code::data().to_vec();
825 let setup = TestSetup {
826 assembly: assembly.clone(),
827 load_addr: GuestAddress(0x1000),
828 initial_regs: Regs {
829 rip: 0x1000,
830 rax: 1,
831 rbx: 2,
832 rflags: 2,
833 ..Default::default()
834 },
835 ..Default::default()
836 };
837
838 run_tests!(
839 setup,
840 |_, regs, _| {
841 assert_eq!(regs.rax, 3); // 1 + 2
842
843 // For VMEXIT caused by HLT, the hypervisor will automatically advance the rIP register.
844 assert_eq!(regs.rip, 0x1000 + assembly.len() as u64);
845 },
846 |_, exit: &_, _: &mut _, _: &mut _| -> bool { matches!(exit, VcpuExit::Hlt) }
847 );
848 }
849
850 global_asm_data!(
851 test_io_exit_handler_code,
852 ".code16",
853 "out 0x10, al",
854 "in al, 0x20",
855 "add ax, bx",
856 "hlt",
857 );
858
859 #[test]
test_io_exit_handler()860 fn test_io_exit_handler() {
861 // Use the OUT/IN instructions, which cause an Io exit in order to
862 // read/write data using a given port.
863 let load_addr = GuestAddress(0x1000);
864 let setup = TestSetup {
865 assembly: test_io_exit_handler_code::data().to_vec(),
866 load_addr,
867 initial_regs: Regs {
868 rip: load_addr.offset(),
869 rax: 0x34, // Only AL (lower byte of RAX) is used
870 rbx: 0x42,
871 rflags: 2,
872 ..Default::default()
873 },
874 ..Default::default()
875 };
876
877 let regs_matcher = |_, regs: &Regs, _: &_| {
878 // The result in AX should be double the initial value of AX
879 // plus the initial value of BX.
880 assert_eq!(regs.rax, (0x34 * 2) + 0x42);
881 };
882
883 let cached_byte = AtomicU8::new(0);
884 let exit_matcher =
885 move |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
886 VcpuExit::Io => {
887 vcpu.handle_io(&mut |IoParams { address, operation }| {
888 match operation {
889 IoOperation::Read(data) => {
890 assert_eq!(address, 0x20);
891 assert_eq!(data.len(), 1);
892 // The original number written below will be doubled and
893 // passed back.
894 data[0] = cached_byte.load(Ordering::SeqCst) * 2;
895 }
896 IoOperation::Write(data) => {
897 assert_eq!(address, 0x10);
898 assert_eq!(data.len(), 1);
899 assert_eq!(data[0], 0x34);
900 cached_byte.fetch_add(data[0], Ordering::SeqCst);
901 }
902 }
903 })
904 .expect("failed to set the data");
905 false // Continue VM runloop
906 }
907 VcpuExit::Hlt => {
908 true // Break VM runloop
909 }
910 r => panic!("unexpected exit reason: {:?}", r),
911 };
912 run_tests!(setup, regs_matcher, &exit_matcher);
913 }
914
915 global_asm_data!(
916 test_io_rep_string_code,
917 ".code16",
918 "cld",
919 "mov dx, 0x80", // read data from I/O port 80h
920 "mov di, 0x100", // write data to memory address 0x100
921 "mov cx, 5", // repeat 5 times
922 "rep insb",
923 "mov si, 0x100", // read data from memory address 0x100
924 "mov dx, 0x80", // write data to I/O port 80h
925 "mov cx, 5", // repeat 5 times
926 "rep outsb",
927 "mov cx, 0x5678",
928 "hlt",
929 );
930
931 #[cfg(not(feature = "haxm"))]
932 #[test]
test_io_rep_string()933 fn test_io_rep_string() {
934 // Test the REP OUTS*/REP INS* string I/O instructions, which should call the IO handler
935 // multiple times to handle the requested repeat count.
936 let load_addr = GuestAddress(0x1000);
937 let setup = TestSetup {
938 assembly: test_io_rep_string_code::data().to_vec(),
939 load_addr,
940 initial_regs: Regs {
941 rip: load_addr.offset(),
942 rax: 0x1234,
943 rflags: 2,
944 ..Default::default()
945 },
946 ..Default::default()
947 };
948
949 let regs_matcher = |_, regs: &Regs, _: &_| {
950 // The string I/O instructions should not modify AX.
951 assert_eq!(regs.rax, 0x1234);
952 assert_eq!(regs.rcx, 0x5678);
953 };
954
955 let read_data = AtomicU8::new(0);
956 let write_data = AtomicU8::new(0);
957 let exit_matcher =
958 move |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| match exit {
959 VcpuExit::Io => {
960 vcpu.handle_io(&mut |IoParams { address, operation }| {
961 match operation {
962 IoOperation::Read(data) => {
963 assert_eq!(address, 0x80);
964 assert_eq!(data.len(), 1);
965 // Return 0, 1, 2, 3, 4 for subsequent reads.
966 data[0] = read_data.fetch_add(1, Ordering::SeqCst);
967 }
968 IoOperation::Write(data) => {
969 assert_eq!(address, 0x80);
970 assert_eq!(data.len(), 1);
971 // Expect 0, 1, 2, 3, 4 to be written.
972 let expected_write = write_data.fetch_add(1, Ordering::SeqCst);
973 assert_eq!(data[0], expected_write);
974 }
975 }
976 })
977 .expect("failed to set the data");
978 false // Continue VM runloop
979 }
980 VcpuExit::Hlt => {
981 // Verify 5 reads and writes occurred.
982 assert_eq!(read_data.load(Ordering::SeqCst), 5);
983 assert_eq!(write_data.load(Ordering::SeqCst), 5);
984
985 // Verify the data that should have been written to memory by REP INSB.
986 let mem = vm.get_memory();
987 let mut data = [0u8; 5];
988 mem.read_exact_at_addr(&mut data, GuestAddress(0x100))
989 .unwrap();
990 assert_eq!(data, [0, 1, 2, 3, 4]);
991
992 true // Break VM runloop
993 }
994 r => panic!("unexpected exit reason: {:?}", r),
995 };
996 run_tests!(setup, regs_matcher, &exit_matcher);
997 }
998
999 global_asm_data!(
1000 test_mmio_exit_cross_page_code,
1001 ".code16",
1002 "mov byte ptr [ebx], al",
1003 "mov al, byte ptr [ecx]",
1004 "hlt",
1005 );
1006
1007 // This test is similar to mmio_fetch_memory.rs (remove eventually)
1008 // but applies to all hypervisors.
1009 #[test]
test_mmio_exit_cross_page()1010 fn test_mmio_exit_cross_page() {
1011 let page_size = 4096u64;
1012 let load_addr = GuestAddress(page_size - 1); // Last byte of the first page
1013
1014 let setup = TestSetup {
1015 assembly: test_mmio_exit_cross_page_code::data().to_vec(),
1016 load_addr,
1017 mem_size: 0x2000,
1018 initial_regs: Regs {
1019 rip: load_addr.offset(),
1020 rax: 0x33,
1021 rbx: 0x3000,
1022 rcx: 0x3010,
1023 rflags: 2,
1024 ..Default::default()
1025 },
1026 ..Default::default()
1027 };
1028
1029 let regs_matcher = |_, regs: &Regs, _: &_| {
1030 assert_eq!(regs.rax, 0x66, "Should match the MMIO read bytes below");
1031 };
1032
1033 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1034 VcpuExit::Mmio => {
1035 vcpu.handle_mmio(&mut |IoParams { address, operation }| {
1036 match operation {
1037 IoOperation::Read(data) => {
1038 match (address, data.len()) {
1039 // First MMIO read asks to load the first 8 bytes
1040 // of a new execution page, when an instruction
1041 // crosses page boundary.
1042 // Return the rest of instructions that are
1043 // supposed to be on the second page.
1044 (0x1000, 8) => {
1045 // Ensure this instruction is the first read
1046 // in the sequence.
1047 data.copy_from_slice(&[0x88, 0x03, 0x67, 0x8a, 0x01, 0xf4, 0, 0]);
1048 Ok(())
1049 }
1050 // Second MMIO read is a regular read from an
1051 // unmapped memory (pointed to by initial EAX).
1052 (0x3010, 1) => {
1053 data.copy_from_slice(&[0x66]);
1054 Ok(())
1055 }
1056 _ => {
1057 panic!("invalid address({:#x})/size({})", address, data.len())
1058 }
1059 }
1060 }
1061 IoOperation::Write(data) => {
1062 assert_eq!(address, 0x3000);
1063 assert_eq!(data[0], 0x33);
1064 assert_eq!(data.len(), 1);
1065 Ok(())
1066 }
1067 }
1068 })
1069 .expect("failed to set the data");
1070 false // Continue VM runloop
1071 }
1072 VcpuExit::Hlt => {
1073 true // Break VM runloop
1074 }
1075 r => panic!("unexpected exit reason: {:?}", r),
1076 };
1077
1078 run_tests!(setup, regs_matcher, exit_matcher);
1079 }
1080
1081 global_asm_data!(
1082 test_mmio_exit_readonly_memory_code,
1083 ".code16",
1084 "mov al,BYTE PTR es:[bx]",
1085 "add al, 0x1",
1086 "mov BYTE PTR es:[bx], al",
1087 "hlt",
1088 );
1089
1090 #[test]
1091 #[cfg(any(target_os = "android", target_os = "linux"))] // Not working for WHXP yet.
test_mmio_exit_readonly_memory()1092 fn test_mmio_exit_readonly_memory() {
1093 // Read from read-only memory and then write back to it,
1094 // which should trigger an MMIO exit.
1095 let setup = TestSetup {
1096 assembly: test_mmio_exit_readonly_memory_code::data().to_vec(),
1097 load_addr: GuestAddress(0x1000),
1098 mem_size: 0x2000,
1099 initial_regs: Regs {
1100 rip: 0x1000,
1101 rax: 1,
1102 rbx: 0,
1103 rflags: 2,
1104 ..Default::default()
1105 },
1106 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
1107 // Add a read-only region of memory to the VM, at address 0x5000.
1108 let prot_mem_size = 0x1000;
1109 let prot_mem =
1110 SharedMemory::new("test", prot_mem_size).expect("failed to create shared memory");
1111 let mmap_ro = MemoryMappingBuilder::new(prot_mem_size as usize)
1112 .from_shared_memory(&prot_mem)
1113 .build()
1114 .expect("failed to create memory mapping");
1115 mmap_ro
1116 .write_obj(0x66, 0)
1117 .expect("failed writing data to ro memory");
1118 vm.add_memory_region(
1119 GuestAddress(0x5000),
1120 Box::new(
1121 MemoryMappingBuilder::new(prot_mem_size as usize)
1122 .from_shared_memory(&prot_mem)
1123 .build()
1124 .expect("failed to create memory mapping"),
1125 ),
1126 true,
1127 false,
1128 CacheCoherent,
1129 )
1130 .expect("failed to register memory");
1131
1132 // Set up segments needed by the assembly addressing above.
1133 let mut sregs = vcpu.get_sregs().expect("get sregs failed");
1134 sregs.cs.s = 1;
1135 sregs.cs.type_ = 0b1011;
1136 sregs.es.base = 0x5000;
1137 sregs.es.selector = 0;
1138 sregs.es.s = 1;
1139 sregs.es.type_ = 0b1011;
1140
1141 vcpu.set_sregs(&sregs).expect("set sregs failed");
1142 })),
1143 ..Default::default()
1144 };
1145
1146 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1147 VcpuExit::Mmio => {
1148 vcpu.handle_mmio(&mut |IoParams { address, operation }| match operation {
1149 IoOperation::Read(_) => {
1150 panic!("unexpected mmio read call");
1151 }
1152 IoOperation::Write(data) => {
1153 assert_eq!(data.len(), 1);
1154 assert_eq!(address, 0x5000);
1155 assert_eq!(data[0], 0x67);
1156 Ok(())
1157 }
1158 })
1159 .expect("failed to set the data");
1160 false // Continue VM runloop
1161 }
1162 VcpuExit::Hlt => {
1163 true // Break VM runloop
1164 }
1165 r => panic!("unexpected exit reason: {:?}", r),
1166 };
1167
1168 run_tests!(
1169 setup,
1170 |_, regs, _| {
1171 assert_eq!(regs.rax, 0x67);
1172 },
1173 exit_matcher
1174 );
1175 }
1176
1177 #[rustfmt::skip::macros(global_asm_data)]
1178 global_asm_data!(
1179 test_cpuid_exit_handler_code,
1180 ".code16",
1181 "cpuid",
1182 "hlt",
1183 );
1184
1185 #[test]
test_cpuid_exit_handler()1186 fn test_cpuid_exit_handler() {
1187 let setup = TestSetup {
1188 assembly: test_cpuid_exit_handler_code::data().to_vec(),
1189 load_addr: GuestAddress(0x1000),
1190 initial_regs: Regs {
1191 rip: 0x1000,
1192 rax: 1, // CPUID input EAX=1 to get virtualization bits.
1193 rflags: 2,
1194 ..Default::default()
1195 },
1196 ..Default::default()
1197 };
1198
1199 let regs_matcher = move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| {
1200 if hypervisor_type == HypervisorType::Haxm {
1201 let hypervisor_bit = regs.rcx & (1 << 31) != 0;
1202 assert!(hypervisor_bit, "Hypervisor bit in CPUID should be set!");
1203 assert_eq!(regs.rip, 0x1003, "CPUID did not execute correctly.");
1204 }
1205 };
1206
1207 let exit_matcher =
1208 |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1209 match hypervisor_type {
1210 HypervisorType::Whpx => match exit {
1211 VcpuExit::Cpuid { entry } => {
1212 println!("Got Cpuid {:?}", entry);
1213 true // Break runloop
1214 }
1215 r => panic!("unexpected exit reason: {:?}", r),
1216 },
1217 _ => match exit {
1218 VcpuExit::Hlt => {
1219 true // Break VM runloop
1220 }
1221 r => panic!("unexpected exit reason: {:?}", r),
1222 },
1223 }
1224 };
1225
1226 run_tests!(setup, regs_matcher, exit_matcher);
1227 }
1228
1229 global_asm_data!(
1230 test_control_register_access_invalid_code,
1231 ".code16",
1232 // Test setting an unused bit in addition to the Protected Mode Enable and Monitor co-processor
1233 // bits, which causes a triple fault and hence the invalid bit should never make it to RCX.
1234 "mov cr0, eax",
1235 "mov ecx, cr0",
1236 "hlt",
1237 );
1238
1239 #[test]
test_control_register_access_invalid()1240 fn test_control_register_access_invalid() {
1241 let setup = TestSetup {
1242 assembly: test_control_register_access_invalid_code::data().to_vec(),
1243 load_addr: GuestAddress(0x1000),
1244 initial_regs: Regs {
1245 rip: 0x1000,
1246 rax: 0x80000011,
1247 rcx: 0,
1248 rflags: 2,
1249 ..Default::default()
1250 },
1251 ..Default::default()
1252 };
1253
1254 // Matcher to check that the RAX value never made it to RCX.
1255 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
1256 assert_eq!(
1257 regs.rcx, 0,
1258 "RCX value mismatch: expected 0, found {:X}",
1259 regs.rcx
1260 )
1261 };
1262
1263 let exit_matcher =
1264 move |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1265 match hypervisor_type {
1266 HypervisorType::Kvm | HypervisorType::Haxm => {
1267 match exit {
1268 VcpuExit::Shutdown(_) => {
1269 true // Break VM runloop
1270 }
1271 r => panic!("unexpected exit reason: {:?}", r),
1272 }
1273 }
1274 _ => {
1275 match exit {
1276 VcpuExit::UnrecoverableException => {
1277 true // Break VM runloop
1278 }
1279 r => panic!("unexpected exit reason: {:?}", r),
1280 }
1281 }
1282 }
1283 };
1284 run_tests!(setup, regs_matcher, exit_matcher);
1285 }
1286
1287 global_asm_data!(
1288 test_control_register_access_valid_code,
1289 // Set the 0th bit (Protected Mode Enable) of CR0, which should succeed.
1290 ".code16",
1291 "mov cr0, eax",
1292 "mov eax, cr0",
1293 "hlt",
1294 );
1295
1296 #[test]
test_control_register_access_valid()1297 fn test_control_register_access_valid() {
1298 let setup = TestSetup {
1299 assembly: test_control_register_access_invalid_code::data().to_vec(),
1300 load_addr: GuestAddress(0x1000),
1301 initial_regs: Regs {
1302 rip: 0x1000,
1303 rax: 0x1,
1304 rflags: 2,
1305 ..Default::default()
1306 },
1307 ..Default::default()
1308 };
1309
1310 // Matcher to check the final state of EAX after reading from CR0
1311 let regs_matcher = |_: HypervisorType, regs: &Regs, _: &_| {
1312 assert!(
1313 (regs.rax & 0x1) != 0,
1314 "CR0 value mismatch: expected the 0th bit to be set, found {:X}",
1315 regs.rax
1316 );
1317 };
1318
1319 let exit_matcher =
1320 move |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1321 VcpuExit::Hlt => {
1322 true // Break VM runloop
1323 }
1324 r => panic!("unexpected exit reason: {:?}", r),
1325 };
1326 run_tests!(setup, regs_matcher, exit_matcher);
1327 }
1328
1329 global_asm_data!(
1330 test_debug_register_access_code,
1331 ".code16",
1332 "mov dr2, eax",
1333 "mov ebx, dr2",
1334 "hlt",
1335 );
1336
1337 #[test]
test_debug_register_access()1338 fn test_debug_register_access() {
1339 let setup = TestSetup {
1340 assembly: test_debug_register_access_code::data().to_vec(),
1341 load_addr: GuestAddress(0x1000),
1342 initial_regs: Regs {
1343 rip: 0x1000,
1344 rax: 0x1234,
1345 rflags: 2,
1346 ..Default::default()
1347 },
1348 ..Default::default()
1349 };
1350
1351 let regs_matcher = |_: HypervisorType, regs: &Regs, _: &_| {
1352 assert_eq!(
1353 regs.rbx, 0x1234,
1354 "DR2 value mismatch: expected 0x1234, found {:X}",
1355 regs.rbx
1356 );
1357 };
1358
1359 let exit_matcher = |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1360 VcpuExit::Hlt => {
1361 true // Break VM runloop
1362 }
1363 r => panic!("unexpected exit reason: {:?}", r),
1364 };
1365
1366 run_tests!(setup, regs_matcher, exit_matcher);
1367 }
1368
1369 // This test only succeeds (by failing Vcpu::Run) on haxm.
1370 #[cfg(all(windows, feature = "haxm"))]
1371 #[test]
test_msr_access_invalid()1372 fn test_msr_access_invalid() {
1373 let msr_index = 0xC0000080; // EFER MSR
1374
1375 let setup = TestSetup {
1376 /*
1377 0: 0f 32 rdmsr
1378 2: 83 c8 02 or ax,0x2 (1st bit is reserved)
1379 5: 0f 30 wrmsr
1380 7: f4 hlt
1381 */
1382 assembly: vec![0x0F, 0x32, 0x83, 0xC8, 0x02, 0x0F, 0x30, 0xF4],
1383 mem_size: 0x5000,
1384 load_addr: GuestAddress(0x1000),
1385 initial_regs: Regs {
1386 rip: 0x1000,
1387 rcx: msr_index, // MSR index to read/write
1388 rflags: 2,
1389 ..Default::default()
1390 },
1391 ..Default::default()
1392 };
1393
1394 let exit_matcher = |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1395 VcpuExit::Shutdown(..) => {
1396 true // Break VM runloop
1397 }
1398 r => panic!("unexpected exit reason: {:?}", r),
1399 };
1400
1401 run_tests!(
1402 setup,
1403 |_, regs, _| {
1404 assert_eq!(regs.rip, 0x1005); // Should stop at the wrmsr
1405 },
1406 exit_matcher
1407 );
1408 }
1409
1410 global_asm_data!(
1411 test_msr_access_valid_code,
1412 ".code16",
1413 "rdmsr",
1414 "add ax, 1",
1415 "wrmsr",
1416 "hlt",
1417 );
1418
1419 #[test]
test_msr_access_valid()1420 fn test_msr_access_valid() {
1421 let msr_index = 0x10; // TSC MSR index
1422
1423 let setup = TestSetup {
1424 assembly: test_msr_access_valid_code::data().to_vec(),
1425 load_addr: GuestAddress(0x1000),
1426 initial_regs: Regs {
1427 rip: 0x1000,
1428 rcx: msr_index, // MSR index for TSC
1429 rflags: 0x2,
1430 ..Default::default()
1431 },
1432 ..Default::default()
1433 };
1434
1435 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
1436 assert!(regs.rax > 0x0, "TSC value should be >0");
1437 assert_eq!(regs.rip, 0x1008, "Should stop after the hlt instruction");
1438 };
1439
1440 let exit_matcher = |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1441 VcpuExit::Hlt => {
1442 true // Break VM runloop
1443 }
1444 r => panic!("unexpected exit reason: {:?}", r),
1445 };
1446 run_tests!(setup, regs_matcher, exit_matcher);
1447 }
1448
1449 #[rustfmt::skip::macros(global_asm_data)]
1450 global_asm_data!(
1451 test_getsec_instruction_code,
1452 ".code16",
1453 "getsec",
1454 "hlt",
1455 );
1456
1457 #[cfg(not(unix))]
1458 #[test]
test_getsec_instruction()1459 fn test_getsec_instruction() {
1460 let setup = TestSetup {
1461 assembly: test_getsec_instruction_code::data().to_vec(),
1462 load_addr: GuestAddress(0x1000),
1463 initial_regs: Regs {
1464 rip: 0x1000,
1465 rflags: 2,
1466 ..Default::default()
1467 },
1468 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
1469 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
1470 })),
1471 ..Default::default()
1472 };
1473
1474 let regs_matcher =
1475 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1476 HypervisorType::Whpx => {}
1477 HypervisorType::Haxm => {}
1478 _ => {
1479 assert_eq!(regs.rip, 0x1000, "GETSEC; expected RIP at 0x1002");
1480 }
1481 };
1482
1483 let exit_matcher =
1484 move |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1485 match hypervisor_type {
1486 HypervisorType::Whpx => {
1487 match exit {
1488 VcpuExit::UnrecoverableException => {
1489 true // Break VM runloop
1490 }
1491 r => panic!("unexpected exit reason: {:?}", r),
1492 }
1493 }
1494 _ => {
1495 match exit {
1496 VcpuExit::Shutdown(_) => {
1497 true // Break VM runloop
1498 }
1499 r => panic!("unexpected exit reason: {:?}", r),
1500 }
1501 }
1502 }
1503 };
1504
1505 run_tests!(setup, regs_matcher, exit_matcher);
1506 }
1507
1508 #[rustfmt::skip::macros(global_asm_data)]
1509 global_asm_data!(
1510 test_invd_instruction_code,
1511 ".code16",
1512 "invd",
1513 "hlt",
1514 );
1515
1516 #[test]
test_invd_instruction()1517 fn test_invd_instruction() {
1518 let setup = TestSetup {
1519 assembly: test_invd_instruction_code::data().to_vec(),
1520 load_addr: GuestAddress(0x1000),
1521 initial_regs: Regs {
1522 rip: 0x1000,
1523 rflags: 2,
1524 ..Default::default()
1525 },
1526 ..Default::default()
1527 };
1528
1529 let regs_matcher =
1530 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1531 HypervisorType::Haxm => {}
1532 _ => {
1533 assert_eq!(regs.rip, 0x1003, "INVD; expected RIP at 0x1003");
1534 }
1535 };
1536 let exit_matcher =
1537 move |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1538 match hypervisor_type {
1539 HypervisorType::Haxm => {
1540 match exit {
1541 VcpuExit::Shutdown(_) => {
1542 true // Break VM runloop
1543 }
1544 r => panic!("unexpected exit reason: {:?}", r),
1545 }
1546 }
1547 _ => {
1548 match exit {
1549 VcpuExit::Hlt => {
1550 true // Break VM runloop
1551 }
1552 r => panic!("unexpected exit reason: {:?}", r),
1553 }
1554 }
1555 }
1556 };
1557
1558 run_tests!(setup, regs_matcher, exit_matcher);
1559 }
1560
1561 global_asm_data!(
1562 test_xsetbv_instruction_code,
1563 ".code16",
1564 "mov eax, cr4",
1565 // Set the OSXSAVE bit in CR4 (bit 9)
1566 "or ax, 0x200",
1567 "mov cr4, eax",
1568 "xgetbv",
1569 "xsetbv",
1570 "hlt",
1571 );
1572
1573 #[test]
test_xsetbv_instruction()1574 fn test_xsetbv_instruction() {
1575 let setup = TestSetup {
1576 assembly: test_xsetbv_instruction_code::data().to_vec(),
1577 load_addr: GuestAddress(0x1000),
1578 initial_regs: Regs {
1579 rip: 0x1000,
1580 rax: 1, // Set bit 0 in EAX
1581 rdx: 0, // XSETBV also uses EDX:EAX, must be initialized
1582 rcx: 0, // XCR0
1583 rflags: 2,
1584 ..Default::default()
1585 },
1586 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
1587 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
1588 })),
1589 ..Default::default()
1590 };
1591
1592 let regs_matcher =
1593 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1594 HypervisorType::Whpx => {}
1595 HypervisorType::Haxm => {}
1596 HypervisorType::Kvm => {}
1597 _ => {
1598 assert_eq!(regs.rip, 0x100D, "XSETBV; expected RIP at 0x100D");
1599 }
1600 };
1601
1602 let exit_matcher = |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1603 match exit {
1604 VcpuExit::Mmio => {
1605 true // Break VM runloop
1606 }
1607 r => panic!("unexpected exit reason: {:?}", r),
1608 }
1609 };
1610
1611 run_tests!(setup, regs_matcher, exit_matcher);
1612 }
1613
1614 global_asm_data!(
1615 test_invept_instruction_code,
1616 ".code16",
1617 "invept eax, [eax]",
1618 "hlt",
1619 );
1620
1621 #[test]
test_invept_instruction()1622 fn test_invept_instruction() {
1623 let setup = TestSetup {
1624 assembly: test_invept_instruction_code::data().to_vec(),
1625 load_addr: GuestAddress(0x1000),
1626 initial_regs: Regs {
1627 rax: 0x2000,
1628 rip: 0x1000,
1629 rflags: 2,
1630 ..Default::default()
1631 },
1632 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
1633 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
1634 })),
1635 ..Default::default()
1636 };
1637
1638 let regs_matcher =
1639 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1640 HypervisorType::Whpx => {}
1641 HypervisorType::Haxm => {}
1642 HypervisorType::Kvm => {}
1643 _ => {
1644 assert_eq!(regs.rip, 0x1005, "invept; expected RIP at 0x1005");
1645 }
1646 };
1647
1648 let exit_matcher =
1649 move |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1650 match hypervisor_type {
1651 HypervisorType::Whpx => {
1652 match exit {
1653 VcpuExit::UnrecoverableException => {
1654 true // Break VM runloop
1655 }
1656 r => panic!("unexpected exit reason: {:?}", r),
1657 }
1658 }
1659 _ => {
1660 match exit {
1661 VcpuExit::Shutdown(_) => {
1662 true // Break VM runloop
1663 }
1664 r => panic!("unexpected exit reason: {:?}", r),
1665 }
1666 }
1667 }
1668 };
1669
1670 run_tests!(setup, regs_matcher, exit_matcher);
1671 }
1672
1673 global_asm_data!(
1674 test_invvpid_instruction_code,
1675 ".code16",
1676 "invvpid eax, [eax]",
1677 "hlt",
1678 );
1679
1680 // TODO(b/342183625): invvpid instruction is not valid in real mode. Reconsider how we should write
1681 // this test.
1682 #[cfg(not(unix))]
1683 #[test]
test_invvpid_instruction()1684 fn test_invvpid_instruction() {
1685 let setup = TestSetup {
1686 assembly: test_invvpid_instruction_code::data().to_vec(),
1687 load_addr: GuestAddress(0x1000),
1688 initial_regs: Regs {
1689 rip: 0x1000,
1690 rax: 0x1500,
1691 rflags: 2,
1692 ..Default::default()
1693 },
1694 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
1695 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
1696 })),
1697 ..Default::default()
1698 };
1699
1700 let regs_matcher = move |_, regs: &Regs, _: &_| {
1701 assert_eq!(regs.rip, 0x1000, "INVVPID; expected RIP at 0x1000");
1702 };
1703
1704 let exit_matcher =
1705 move |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
1706 VcpuExit::Mmio | VcpuExit::Shutdown(_) | VcpuExit::InternalError => {
1707 true // Break VM runloop
1708 }
1709 r => panic!("unexpected exit reason: {:?}", r),
1710 };
1711
1712 run_tests!(setup, regs_matcher, exit_matcher);
1713 }
1714
1715 #[test]
test_vm_instruction_set()1716 fn test_vm_instruction_set() {
1717 let instructions = vec![
1718 (vec![0x0F, 0x01, 0xC1], 0x1000, "VMCALL"), // VMCALL
1719 (vec![0x66, 0x0F, 0xC7, 0x30], 0x1004, "VMCLEAR"), // VMCLEAR
1720 (vec![0x0F, 0x01, 0xC2], 0x1003, "VMLAUNCH"), // VMLAUNCH
1721 (vec![0x0F, 0xC7, 0x30], 0x1003, "VMPTRLD"), // VMPTRLD
1722 (vec![0x0F, 0xC7, 0x31], 0x1003, "VMPTRST"), // VMPTRST
1723 (vec![0x0F, 0x01, 0xC3], 0x1003, "VMRESUME"), // VMRESUME
1724 (vec![0x0F, 0x01, 0xC4], 0x1003, "VMXOFF"), // VMXOFF
1725 (vec![0x0F, 0x01, 0xC4], 0x1003, "VMXON"), // VMXON
1726 ];
1727
1728 for (bytes, expected_rip, name) in instructions {
1729 let mut assembly = bytes;
1730 assembly.push(0xF4); // Append HLT to each instruction set
1731
1732 let setup = TestSetup {
1733 assembly,
1734 load_addr: GuestAddress(0x1000),
1735 initial_regs: Regs {
1736 rip: 0x1000,
1737 rflags: 2,
1738 ..Default::default()
1739 },
1740 ..Default::default()
1741 };
1742
1743 let regs_matcher =
1744 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1745 HypervisorType::Whpx => {}
1746 HypervisorType::Kvm => {}
1747 HypervisorType::Haxm => {}
1748 _ => {
1749 assert_eq!(
1750 regs.rip, expected_rip,
1751 "{}; expected RIP at {}",
1752 name, expected_rip
1753 );
1754 }
1755 };
1756
1757 let exit_matcher =
1758 |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1759 match hypervisor_type {
1760 HypervisorType::Whpx => {
1761 match exit {
1762 VcpuExit::Mmio => {
1763 true // Break VM runloop
1764 }
1765 r => panic!("unexpected exit reason: {:?}", r),
1766 }
1767 }
1768 HypervisorType::Kvm => {
1769 true // Break VM runloop
1770 }
1771 _ => {
1772 match exit {
1773 VcpuExit::Shutdown(_) => {
1774 true // Break VM runloop
1775 }
1776 r => panic!("unexpected exit reason: {:?}", r),
1777 }
1778 }
1779 }
1780 };
1781
1782 run_tests!(setup, regs_matcher, exit_matcher);
1783 }
1784 }
1785
1786 #[rustfmt::skip::macros(global_asm_data)]
1787 global_asm_data!(
1788 test_software_interrupt_code,
1789 "int 0x80",
1790 "hlt",
1791 );
1792
1793 #[test]
test_software_interrupt()1794 fn test_software_interrupt() {
1795 let start_addr = 0x1000;
1796 let setup = TestSetup {
1797 assembly: test_software_interrupt_code::data().to_vec(),
1798 load_addr: GuestAddress(0x1000),
1799 initial_regs: Regs {
1800 rip: start_addr,
1801 rflags: 2,
1802 ..Default::default()
1803 },
1804 ..Default::default()
1805 };
1806
1807 let regs_matcher =
1808 move |hypervisor_type: HypervisorType, regs: &Regs, _: &_| match hypervisor_type {
1809 HypervisorType::Whpx => {}
1810 HypervisorType::Haxm => {}
1811 HypervisorType::Kvm => {}
1812 _ => {
1813 let expect_rip_addr = start_addr
1814 + u64::try_from(test_software_interrupt_code::data().len())
1815 .expect("the code length should within the range of u64");
1816 assert_eq!(
1817 regs.rip, expect_rip_addr,
1818 "Expected RIP at {:#x}",
1819 expect_rip_addr
1820 );
1821 }
1822 };
1823
1824 let exit_matcher =
1825 |hypervisor_type, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
1826 match hypervisor_type {
1827 HypervisorType::Kvm | HypervisorType::Whpx => {
1828 match exit {
1829 VcpuExit::Mmio => {
1830 true // Break VM runloop
1831 }
1832 r => panic!("unexpected exit reason: {:?}", r),
1833 }
1834 }
1835 _ => {
1836 match exit {
1837 VcpuExit::Shutdown(_) => {
1838 true // Break VM runloop
1839 }
1840 r => panic!("unexpected exit reason: {:?}", r),
1841 }
1842 }
1843 }
1844 };
1845
1846 run_tests!(setup, regs_matcher, exit_matcher);
1847 }
1848
1849 #[rustfmt::skip::macros(global_asm_data)]
1850 global_asm_data!(
1851 test_rdtsc_instruction_code,
1852 ".code16",
1853 "rdtsc",
1854 "hlt",
1855 );
1856
1857 #[test]
test_rdtsc_instruction()1858 fn test_rdtsc_instruction() {
1859 let setup = TestSetup {
1860 assembly: test_rdtsc_instruction_code::data().to_vec(),
1861 load_addr: GuestAddress(0x1000),
1862 initial_regs: Regs {
1863 rip: 0x1000,
1864 rflags: 2,
1865 ..Default::default()
1866 },
1867 ..Default::default()
1868 };
1869
1870 // This matcher checks that the timestamp counter has been incremented and read into EAX and EDX
1871 let regs_matcher = |_: HypervisorType, regs: &Regs, _: &_| {
1872 assert!(
1873 regs.rax != 0 || regs.rdx != 0,
1874 "RDTSC returned a zero value, which is unlikely."
1875 );
1876 };
1877
1878 let exit_matcher = |_: HypervisorType,
1879 exit: &VcpuExit,
1880 _: &mut dyn VcpuX86_64,
1881 _: &mut dyn Vm| { matches!(exit, VcpuExit::Hlt) };
1882
1883 run_tests!(setup, regs_matcher, exit_matcher);
1884 }
1885
1886 global_asm_data!(
1887 test_register_access_code,
1888 ".code16",
1889 "xchg ax, bx",
1890 "xchg cx, dx",
1891 "xchg sp, bp",
1892 "xchg si, di",
1893 "hlt",
1894 );
1895
1896 // This tests that we can write and read GPRs to/from the VM.
1897 #[test]
test_register_access()1898 fn test_register_access() {
1899 let start_addr = 0x1000;
1900 let setup = TestSetup {
1901 assembly: test_register_access_code::data().to_vec(),
1902 load_addr: GuestAddress(start_addr),
1903 initial_regs: Regs {
1904 rip: start_addr,
1905 rax: 2,
1906 rbx: 1,
1907 rcx: 4,
1908 rdx: 3,
1909 rsp: 6,
1910 rbp: 5,
1911 rsi: 8,
1912 rdi: 7,
1913 rflags: 2,
1914 ..Default::default()
1915 },
1916 ..Default::default()
1917 };
1918
1919 run_tests!(
1920 setup,
1921 |_, regs, _| {
1922 assert_eq!(regs.rax, 1);
1923 assert_eq!(regs.rbx, 2);
1924 assert_eq!(regs.rcx, 3);
1925 assert_eq!(regs.rdx, 4);
1926 assert_eq!(regs.rsp, 5);
1927 assert_eq!(regs.rbp, 6);
1928 assert_eq!(regs.rsi, 7);
1929 assert_eq!(regs.rdi, 8);
1930 assert_eq!(
1931 regs.rip,
1932 start_addr + test_register_access_code::data().len() as u64
1933 );
1934 },
1935 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
1936 );
1937 }
1938
1939 global_asm_data!(
1940 test_flags_register_code,
1941 ".code16",
1942 "jnz fin",
1943 "test ax, ax",
1944 "fin:",
1945 "hlt",
1946 );
1947
1948 // This tests that we can get/set the flags register from the VMM.
1949 #[test]
test_flags_register()1950 fn test_flags_register() {
1951 let start_addr = 0x1000;
1952 let setup = TestSetup {
1953 assembly: test_flags_register_code::data().to_vec(),
1954 load_addr: GuestAddress(start_addr),
1955 initial_regs: Regs {
1956 rip: start_addr,
1957 rax: 0xffffffff,
1958 rflags: 0x42, // zero flag set, sign flag clear
1959 ..Default::default()
1960 },
1961 ..Default::default()
1962 };
1963
1964 run_tests!(
1965 setup,
1966 |_, regs, _| {
1967 assert_eq!(regs.rflags & 0x40, 0); // zero flag is clear
1968 assert_ne!(regs.rflags & 0x80, 0); // sign flag is set
1969 assert_eq!(
1970 regs.rip,
1971 start_addr + test_flags_register_code::data().len() as u64
1972 );
1973 },
1974 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
1975 );
1976 }
1977
1978 global_asm_data!(
1979 test_vmm_set_segs_code,
1980 ".code16",
1981 "mov ax, ds:0",
1982 "mov bx, es:0",
1983 "mov cx, fs:0",
1984 "mov dx, gs:0",
1985 "mov sp, ss:0",
1986 "hlt",
1987 );
1988
1989 // This tests that the VMM can set segment registers and have them used by the VM.
1990 #[test]
test_vmm_set_segs()1991 fn test_vmm_set_segs() {
1992 let start_addr = 0x1000;
1993 let data_addr = 0x2000;
1994 let setup = TestSetup {
1995 assembly: test_vmm_set_segs_code::data().to_vec(),
1996 load_addr: GuestAddress(start_addr),
1997 mem_size: 0x4000,
1998 initial_regs: Regs {
1999 rip: start_addr,
2000 rflags: 0x42,
2001 ..Default::default()
2002 },
2003 // simple memory pattern where the value of a byte is (addr - data_addr + 1)
2004 memory_initializations: vec![(GuestAddress(data_addr), (1..=32).collect())],
2005 extra_vm_setup: Some(Box::new(move |vcpu: &mut dyn VcpuX86_64, _| {
2006 let mut sregs = vcpu.get_sregs().expect("failed to get sregs");
2007 sregs.ds.base = data_addr;
2008 sregs.ds.selector = 0;
2009 sregs.es.base = data_addr + 4;
2010 sregs.es.selector = 0;
2011 sregs.fs.base = data_addr + 8;
2012 sregs.fs.selector = 0;
2013 sregs.gs.base = data_addr + 12;
2014 sregs.gs.selector = 0;
2015 sregs.ss.base = data_addr + 16;
2016 sregs.ss.selector = 0;
2017 vcpu.set_sregs(&sregs).expect("failed to set sregs");
2018 })),
2019 ..Default::default()
2020 };
2021
2022 run_tests!(
2023 setup,
2024 |_, regs, sregs| {
2025 assert_eq!(sregs.ds.base, data_addr);
2026 assert_eq!(sregs.es.base, data_addr + 4);
2027 assert_eq!(sregs.fs.base, data_addr + 8);
2028 assert_eq!(sregs.gs.base, data_addr + 12);
2029 assert_eq!(sregs.ss.base, data_addr + 16);
2030
2031 // ax was loaded from ds:0, which has offset 0, so is [1, 2]
2032 assert_eq!(regs.rax, 0x0201);
2033 // bx was loaded from es:0, which has offset 4, so is [5, 6]
2034 assert_eq!(regs.rbx, 0x0605);
2035 // cx was loaded from fs:0, which has offset 8, so is [9, 10]
2036 assert_eq!(regs.rcx, 0x0a09);
2037 // dx was loaded from gs:0, which has offset 12, so is [13, 14]
2038 assert_eq!(regs.rdx, 0x0e0d);
2039 // sp was loaded from ss:0, which has offset 16, so is [17, 18]
2040 assert_eq!(regs.rsp, 0x1211);
2041
2042 let expect_rip_addr = start_addr
2043 + u64::try_from(test_vmm_set_segs_code::data().len())
2044 .expect("the code length should within the range of u64");
2045 assert_eq!(
2046 regs.rip, expect_rip_addr,
2047 "Expected RIP at {:#x}",
2048 expect_rip_addr
2049 );
2050 },
2051 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
2052 );
2053 }
2054
2055 global_asm_data!(
2056 test_set_cr_vmm_code,
2057 ".code16",
2058 "mov eax, cr0",
2059 "mov ebx, cr3",
2060 "mov ecx, cr4",
2061 "hlt",
2062 );
2063
2064 // Tests that the VMM can read and write CRs and they become visible in the guest.
2065 #[test]
test_set_cr_vmm()2066 fn test_set_cr_vmm() {
2067 let asm_addr = 0x1000;
2068 let setup = TestSetup {
2069 assembly: test_set_cr_vmm_code::data().to_vec(),
2070 load_addr: GuestAddress(asm_addr),
2071 initial_regs: Regs {
2072 rip: asm_addr,
2073 rflags: 2,
2074 ..Default::default()
2075 },
2076 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, _| {
2077 let mut sregs = vcpu.get_sregs().expect("failed to get sregs");
2078 sregs.cr0 |= 1 << 18; // Alignment Mask; does nothing without other config bits
2079 sregs.cr3 = 0xfeedface; // arbitrary value; CR3 is not used in this configuration
2080 sregs.cr4 |= 1 << 2; // Time Stamp Disable; not relevant here
2081 vcpu.set_sregs(&sregs).expect("failed to set sregs");
2082 })),
2083 ..Default::default()
2084 };
2085
2086 run_tests!(
2087 setup,
2088 |_, regs, sregs| {
2089 assert_eq!(regs.rax, sregs.cr0);
2090 assert_eq!(regs.rbx, sregs.cr3);
2091 assert_eq!(regs.rcx, sregs.cr4);
2092 assert_eq!(sregs.cr3, 0xfeedface);
2093 assert_ne!(sregs.cr0 & (1 << 18), 0);
2094 assert_ne!(sregs.cr4 & (1 << 2), 0);
2095 assert_eq!(regs.rip, asm_addr + setup.assembly.len() as u64); // after hlt
2096 },
2097 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
2098 );
2099 }
2100
2101 global_asm_data!(
2102 test_set_cr_guest_code,
2103 ".code16",
2104 "mov eax, cr0",
2105 "or eax, (1 << 18)",
2106 "mov cr0, eax",
2107 "mov ebx, 0xfeedface",
2108 "mov cr3, ebx",
2109 "mov ecx, cr4",
2110 "or ecx, (1 << 2)",
2111 "mov cr4, ecx",
2112 "hlt",
2113 );
2114
2115 // Tests that the guest can read and write CRs and they become visible to the VMM.
2116 #[test]
test_set_cr_guest()2117 fn test_set_cr_guest() {
2118 let asm_addr = 0x1000;
2119 let setup = TestSetup {
2120 assembly: test_set_cr_guest_code::data().to_vec(),
2121 load_addr: GuestAddress(asm_addr),
2122 initial_regs: Regs {
2123 rip: asm_addr,
2124 rflags: 2,
2125 ..Default::default()
2126 },
2127 ..Default::default()
2128 };
2129
2130 run_tests!(
2131 setup,
2132 |_, regs, sregs| {
2133 assert_eq!(regs.rax, sregs.cr0);
2134 assert_eq!(regs.rbx, sregs.cr3);
2135 assert_eq!(regs.rcx, sregs.cr4);
2136 assert_eq!(sregs.cr3, 0xfeedface);
2137 assert_ne!(sregs.cr0 & (1 << 18), 0);
2138 assert_ne!(sregs.cr4 & (1 << 2), 0);
2139 assert_eq!(regs.rip, asm_addr + setup.assembly.len() as u64); // after hlt
2140 },
2141 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
2142 );
2143 }
2144
2145 mod test_minimal_interrupt_injection_code {
2146 use super::*;
2147
2148 global_asm_data!(
2149 pub init,
2150 ".code16",
2151 // Set the IDT
2152 "lidt [0x200]",
2153 // Set up the stack, which will be used when CPU transfers the control to the ISR on
2154 // interrupt.
2155 "mov sp, 0x900",
2156 "mov eax, 902",
2157 // We inject our exception on this hlt command.
2158 "hlt",
2159 "mov ebx, 990",
2160 "hlt"
2161 );
2162
2163 global_asm_data!(
2164 pub isr,
2165 ".code16",
2166 "mov eax, 888",
2167 "iret"
2168 );
2169 }
2170
2171 #[test]
test_minimal_interrupt_injection()2172 fn test_minimal_interrupt_injection() {
2173 let start_addr: u32 = 0x200;
2174 // Allocate exceed 0x900, where we set up our stack.
2175 let mem_size: u32 = 0x1000;
2176
2177 let mut setup = TestSetup {
2178 load_addr: GuestAddress(start_addr.into()),
2179 initial_regs: Regs {
2180 rax: 0,
2181 rbx: 0,
2182 // Set RFLAGS.IF to enable interrupt.
2183 rflags: 2 | FLAGS_IF_BIT,
2184 ..Default::default()
2185 },
2186 mem_size: mem_size.into(),
2187 ..Default::default()
2188 };
2189
2190 let mut cur_addr = start_addr;
2191
2192 let idtr_size: u32 = 6;
2193 assert_eq!(
2194 Ok(std::mem::size_of::<Idtr32>()),
2195 usize::try_from(idtr_size)
2196 );
2197 // The limit is calculated from 256 entries timed by 4 bytes per entry.
2198 let idt_size = 256u16 * 4u16;
2199 let idtr = Idtr32 {
2200 limit: idt_size - 1,
2201 // The IDT right follows the IDTR.
2202 base_address: start_addr + idtr_size,
2203 };
2204 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idtr.as_bytes().to_vec());
2205 cur_addr += idtr_size;
2206
2207 let idt_entry = (start_addr + idtr_size + u32::from(idt_size)).to_ne_bytes();
2208 // IDT entries are far pointers(CS:IP pair) to the only ISR, which locates right after the IDT.
2209 // We set all entries to the same ISR.
2210 let idt = (0..256).flat_map(|_| idt_entry).collect::<Vec<_>>();
2211 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idt.clone());
2212 cur_addr += u32::try_from(idt.len()).expect("IDT size should be within u32");
2213
2214 let isr_assembly = test_minimal_interrupt_injection_code::isr::data().to_vec();
2215 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_assembly.clone());
2216 cur_addr += u32::try_from(isr_assembly.len()).expect("ISR size should be within u32");
2217
2218 let init_assembly = test_minimal_interrupt_injection_code::init::data().to_vec();
2219 setup.initial_regs.rip = cur_addr.into();
2220 setup.add_memory_initialization(GuestAddress(cur_addr.into()), init_assembly.clone());
2221 cur_addr += u32::try_from(init_assembly.len()).expect("init size should be within u32");
2222 let init_end_addr = cur_addr;
2223
2224 assert!(mem_size > cur_addr);
2225
2226 let mut counter = 0;
2227 run_tests!(
2228 setup,
2229 |_, regs, _| {
2230 assert_eq!(regs.rip, u64::from(init_end_addr));
2231 assert_eq!(regs.rax, 888);
2232 assert_eq!(regs.rbx, 990);
2233 },
2234 |_, exit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
2235 match exit {
2236 VcpuExit::Hlt => {
2237 let regs = vcpu
2238 .get_regs()
2239 .expect("should retrieve registers successfully");
2240 counter += 1;
2241 if counter > 1 {
2242 return true;
2243 }
2244 assert!(vcpu.ready_for_interrupt());
2245 assert_eq!(regs.rax, 902);
2246 assert_eq!(regs.rbx, 0);
2247 // Inject an external custom interrupt.
2248 vcpu.interrupt(32)
2249 .expect("should be able to inject an interrupt");
2250 false
2251 }
2252 r => panic!("unexpected VMEXIT reason: {:?}", r),
2253 }
2254 }
2255 );
2256 }
2257
2258 mod test_multiple_interrupt_injection_code {
2259 use super::*;
2260
2261 global_asm_data!(
2262 pub init,
2263 ".code16",
2264 // Set the IDT
2265 "lidt [0x200]",
2266 // Set up the stack, which will be used when CPU transfers the control to the ISR on
2267 // interrupt.
2268 "mov esp, 0x900",
2269 "mov eax, 1",
2270 "mov ebx, 2",
2271 "mov ecx, 3",
2272 "mov edx, 4",
2273 // We inject our interrupts on this hlt command.
2274 "hlt",
2275 "mov edx, 281",
2276 "hlt",
2277 );
2278
2279 global_asm_data!(
2280 pub isr_intr_32,
2281 ".code16",
2282 "mov eax, 32",
2283 "iret",
2284 );
2285
2286 global_asm_data!(
2287 pub isr_intr_33,
2288 ".code16",
2289 "mov ebx, 33",
2290 "iret",
2291 );
2292
2293 global_asm_data!(
2294 pub isr_default,
2295 ".code16",
2296 "mov ecx, 761",
2297 "iret",
2298 );
2299 }
2300
2301 #[test]
test_multiple_interrupt_injection()2302 fn test_multiple_interrupt_injection() {
2303 let start_addr: u32 = 0x200;
2304 // Allocate exceed 0x900, where we set up our stack.
2305 let mem_size: u32 = 0x1000;
2306
2307 let mut setup = TestSetup {
2308 load_addr: GuestAddress(start_addr.into()),
2309 initial_regs: Regs {
2310 rax: 0,
2311 rbx: 0,
2312 rcx: 0,
2313 rdx: 0,
2314 // Set RFLAGS.IF to enable interrupt.
2315 rflags: 2 | FLAGS_IF_BIT,
2316 ..Default::default()
2317 },
2318 mem_size: mem_size.into(),
2319 ..Default::default()
2320 };
2321
2322 let mut cur_addr = start_addr;
2323
2324 let idtr_size: u32 = 6;
2325 assert_eq!(
2326 Ok(std::mem::size_of::<Idtr32>()),
2327 usize::try_from(idtr_size)
2328 );
2329 // The limit is calculated from 256 entries timed by 4 bytes per entry.
2330 let idt_size = 256u16 * 4u16;
2331 let idtr = Idtr32 {
2332 limit: idt_size - 1,
2333 // The IDT right follows the IDTR.
2334 base_address: start_addr + idtr_size,
2335 };
2336 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idtr.as_bytes().to_vec());
2337 cur_addr += idtr_size;
2338
2339 let isr_intr_32_assembly = test_multiple_interrupt_injection_code::isr_intr_32::data().to_vec();
2340 let isr_intr_33_assembly = test_multiple_interrupt_injection_code::isr_intr_33::data().to_vec();
2341 let isr_default_assembly = test_multiple_interrupt_injection_code::isr_default::data().to_vec();
2342 // The ISR for intr 32 right follows the IDT.
2343 let isr_intr_32_addr = cur_addr + u32::from(idt_size);
2344 // The ISR for intr 33 right follows the ISR for intr 32.
2345 let isr_intr_33_addr = isr_intr_32_addr
2346 + u32::try_from(isr_intr_32_assembly.len())
2347 .expect("the size of the ISR for intr 32 should be within the u32 range");
2348 // The ISR for other interrupts right follows the ISR for intr 33.
2349 let isr_default_addr = isr_intr_33_addr
2350 + u32::try_from(isr_intr_33_assembly.len())
2351 .expect("the size of the ISR for intr 33 should be within the u32 range");
2352
2353 // IDT entries are far pointers(CS:IP pair) to the correspondent ISR.
2354 let idt = (0..256)
2355 .map(|intr_vec| match intr_vec {
2356 32 => isr_intr_32_addr,
2357 33 => isr_intr_33_addr,
2358 _ => isr_default_addr,
2359 })
2360 .flat_map(u32::to_ne_bytes)
2361 .collect::<Vec<_>>();
2362 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idt.clone());
2363 assert_eq!(idt.len(), usize::from(idt_size));
2364 cur_addr += u32::try_from(idt.len()).expect("IDT size should be within u32");
2365
2366 assert_eq!(cur_addr, isr_intr_32_addr);
2367 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_intr_32_assembly.clone());
2368 cur_addr += u32::try_from(isr_intr_32_assembly.len()).expect("ISR size should be within u32");
2369
2370 assert_eq!(cur_addr, isr_intr_33_addr);
2371 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_intr_33_assembly.clone());
2372 cur_addr += u32::try_from(isr_intr_33_assembly.len()).expect("ISR size should be within u32");
2373
2374 assert_eq!(cur_addr, isr_default_addr);
2375 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_default_assembly.clone());
2376 cur_addr += u32::try_from(isr_default_assembly.len()).expect("ISR size should be within u32");
2377
2378 let init_assembly = test_multiple_interrupt_injection_code::init::data().to_vec();
2379 setup.initial_regs.rip = cur_addr.into();
2380 setup.add_memory_initialization(GuestAddress(cur_addr.into()), init_assembly.clone());
2381 cur_addr += u32::try_from(init_assembly.len()).expect("init size should be within u32");
2382 let init_end_addr = cur_addr;
2383
2384 assert!(mem_size > cur_addr);
2385
2386 let mut counter = 0;
2387 run_tests!(
2388 setup,
2389 |hypervisor_type, regs, _| {
2390 // Different hypervisors behave differently on how the first injected exception should
2391 // handled: for WHPX and KVM, the later injected interrupt overrides the earlier
2392 // injected interrupt, while for HAXM, both interrupts are marked as pending.
2393 match hypervisor_type {
2394 HypervisorType::Haxm => assert_eq!(regs.rax, 32),
2395 _ => assert_eq!(regs.rax, 1),
2396 }
2397
2398 assert_eq!(regs.rip, u64::from(init_end_addr));
2399 assert_eq!(regs.rbx, 33);
2400 assert_eq!(regs.rcx, 3);
2401 assert_eq!(regs.rdx, 281);
2402 },
2403 |_, exit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
2404 match exit {
2405 VcpuExit::Hlt => {
2406 let regs = vcpu
2407 .get_regs()
2408 .expect("should retrieve registers successfully");
2409 counter += 1;
2410 if counter > 1 {
2411 return true;
2412 }
2413 assert_eq!(regs.rax, 1);
2414 assert_eq!(regs.rbx, 2);
2415 assert_eq!(regs.rcx, 3);
2416 assert_eq!(regs.rdx, 4);
2417 // Inject external custom interrupts.
2418 assert!(vcpu.ready_for_interrupt());
2419 vcpu.interrupt(32)
2420 .expect("should be able to inject an interrupt");
2421 assert!(vcpu.ready_for_interrupt());
2422 vcpu.interrupt(33)
2423 .expect("should be able to inject an interrupt");
2424 false
2425 }
2426 r => panic!("unexpected VMEXIT reason: {:?}", r),
2427 }
2428 }
2429 );
2430 }
2431
2432 mod test_interrupt_ready_when_not_interruptible_code {
2433 use super::*;
2434
2435 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
2436 pub enum Instrumentation {
2437 BeforeMovSs,
2438 AfterMovSs,
2439 AfterAfterMovSs,
2440 BeforeSti,
2441 AfterSti,
2442 AfterAfterSti,
2443 InIsr,
2444 }
2445
2446 impl From<u64> for Instrumentation {
from(value: u64) -> Self2447 fn from(value: u64) -> Self {
2448 match value {
2449 0x10 => Instrumentation::BeforeMovSs,
2450 0x20 => Instrumentation::AfterMovSs,
2451 0x30 => Instrumentation::AfterAfterMovSs,
2452 0x40 => Instrumentation::BeforeSti,
2453 0x50 => Instrumentation::AfterSti,
2454 0x60 => Instrumentation::AfterAfterSti,
2455 0xf0 => Instrumentation::InIsr,
2456 _ => panic!("Unknown instrumentation IO port: {}", value),
2457 }
2458 }
2459 }
2460
2461 // We use port IO to trigger the VMEXIT instead of MMIO, because access to out of bound memory
2462 // doesn't trigger MMIO VMEXIT on WHPX under simple real-mode set up.
2463 global_asm_data!(
2464 pub init,
2465 ".code16",
2466 // Set up the stack, which will be used when CPU transfers the control to the ISR on
2467 // interrupt.
2468 "mov sp, 0x1900",
2469 // Set the IDT.
2470 "lidt [0x200]",
2471 // Load the ss register, so that the later mov ss instruction is actually a no-op.
2472 "mov ax, ss",
2473 "out 0x10, ax",
2474 // Hypervisors shouldn't allow interrupt injection right after the mov ss instruction.
2475 "mov ss, ax",
2476 "out 0x20, ax",
2477 // On WHPX we need some other instructions to bring the interuptibility back to normal.
2478 // While this is not needed for other hypervisors, we add this instruction unconditionally.
2479 "nop",
2480 "out 0x30, ax",
2481 "out 0x40, ax",
2482 // Test hypervisors' interruptibilities right after sti instruction when FLAGS.IF is
2483 // cleared.
2484 "cli",
2485 "sti",
2486 "out 0x50, ax",
2487 // On WHPX we need some other instructions to bring the interuptibility back to normal.
2488 // While this is not needed for other hypervisors, we add this instruction unconditionally.
2489 "nop",
2490 "out 0x60, ax",
2491 "hlt",
2492 );
2493
2494 global_asm_data!(
2495 pub isr,
2496 ".code16",
2497 "out 0xf0, ax",
2498 "iret",
2499 );
2500 }
2501
2502 // Physical x86 processor won't allow interrupt to be injected after mov ss or sti, while VM can.
2503 #[test]
test_interrupt_ready_when_normally_not_interruptible()2504 fn test_interrupt_ready_when_normally_not_interruptible() {
2505 use test_interrupt_ready_when_not_interruptible_code::Instrumentation;
2506
2507 let start_addr: u32 = 0x200;
2508 // Allocate exceed 0x1900, where we set up our stack.
2509 let mem_size: u32 = 0x2000;
2510
2511 let mut setup = TestSetup {
2512 load_addr: GuestAddress(start_addr.into()),
2513 initial_regs: Regs {
2514 rax: 0,
2515 rbx: 0,
2516 // Set RFLAGS.IF to enable interrupt.
2517 rflags: 2 | 0x202,
2518 ..Default::default()
2519 },
2520 mem_size: mem_size.into(),
2521 ..Default::default()
2522 };
2523
2524 let mut cur_addr = start_addr;
2525
2526 let idtr_size: u32 = 6;
2527 assert_eq!(
2528 Ok(std::mem::size_of::<Idtr32>()),
2529 usize::try_from(idtr_size)
2530 );
2531 // The limit is calculated from 256 entries timed by 4 bytes per entry.
2532 let idt_size = 256u16 * 4u16;
2533 let idtr = Idtr32 {
2534 limit: idt_size - 1,
2535 // The IDT right follows the IDTR.
2536 base_address: start_addr + idtr_size,
2537 };
2538 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idtr.as_bytes().to_vec());
2539 cur_addr += idtr_size;
2540
2541 let idt_entry = (start_addr + idtr_size + u32::from(idt_size)).to_ne_bytes();
2542 // IDT entries are far pointers(CS:IP pair) to the only ISR, which locates right after the IDT.
2543 // We set all entries to the same ISR.
2544 let idt = (0..256).flat_map(|_| idt_entry).collect::<Vec<_>>();
2545 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idt.clone());
2546 cur_addr += u32::try_from(idt.len()).expect("IDT size should be within u32");
2547
2548 let isr_assembly = test_interrupt_ready_when_not_interruptible_code::isr::data().to_vec();
2549 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_assembly.clone());
2550 cur_addr += u32::try_from(isr_assembly.len()).expect("ISR size should be within u32");
2551
2552 let init_assembly = test_interrupt_ready_when_not_interruptible_code::init::data().to_vec();
2553 setup.initial_regs.rip = cur_addr.into();
2554 setup.add_memory_initialization(GuestAddress(cur_addr.into()), init_assembly.clone());
2555 cur_addr += u32::try_from(init_assembly.len()).expect("init size should be within u32");
2556
2557 assert!(mem_size > cur_addr);
2558
2559 // This helps us check the interruptibility under different situations.
2560 let interruptibility_traces = RefCell::<Vec<_>>::default();
2561 // This helps us check when the interrupt actually delivers.
2562 let instrumentation_traces = RefCell::<Vec<_>>::default();
2563
2564 run_tests!(
2565 setup,
2566 |_, regs, _| {
2567 use Instrumentation::*;
2568 assert_eq!(
2569 *interruptibility_traces.borrow(),
2570 [
2571 (BeforeMovSs, true),
2572 // Hypervisors don't allow interrupt injection right after mov ss.
2573 (AfterMovSs, false),
2574 (AfterAfterMovSs, true),
2575 (BeforeSti, true),
2576 // Hypervisors don't allow interrupt injection right after sti when FLAGS.IF is
2577 // not set.
2578 (AfterSti, false),
2579 (AfterAfterSti, true)
2580 ]
2581 );
2582 // Hypervisors always deliver the interrupt right after we inject it in the next VCPU
2583 // run.
2584 assert_eq!(
2585 *instrumentation_traces.borrow(),
2586 [
2587 BeforeMovSs,
2588 InIsr,
2589 AfterMovSs,
2590 AfterAfterMovSs,
2591 InIsr,
2592 BeforeSti,
2593 InIsr,
2594 AfterSti,
2595 AfterAfterSti,
2596 InIsr,
2597 ]
2598 );
2599 assert_eq!(regs.rip, u64::from(cur_addr));
2600 },
2601 |_, exit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
2602 match exit {
2603 VcpuExit::Io => {
2604 let ready_for_interrupt = vcpu.ready_for_interrupt();
2605 let mut should_inject_interrupt = ready_for_interrupt;
2606 vcpu.handle_io(&mut |io_params| {
2607 let instrumentation = Instrumentation::from(io_params.address);
2608 match instrumentation {
2609 Instrumentation::InIsr => {
2610 // Only inject interrupt outside ISR.
2611 should_inject_interrupt = false;
2612 }
2613 _ => {
2614 // Only the interuptibility outside the ISR is important for this
2615 // test.
2616 interruptibility_traces
2617 .borrow_mut()
2618 .push((instrumentation, ready_for_interrupt));
2619 }
2620 }
2621 instrumentation_traces.borrow_mut().push(instrumentation);
2622 // We are always handling out IO port, so no data to return.
2623 })
2624 .expect("should handle IO successfully");
2625 if should_inject_interrupt {
2626 vcpu.interrupt(32)
2627 .expect("interrupt injection should succeed when ready for interrupt");
2628 }
2629 false
2630 }
2631 VcpuExit::Hlt => true,
2632 r => panic!("unexpected VMEXIT reason: {:?}", r),
2633 }
2634 }
2635 );
2636 }
2637
2638 global_asm_data!(
2639 test_interrupt_ready_when_interrupt_enable_flag_not_set_code,
2640 ".code16",
2641 "cli",
2642 // We can't use hlt for VMEXIT, because HAXM unconditionally allows interrupt injection for
2643 // hlt.
2644 "out 0x10, ax",
2645 "sti",
2646 // nop is necessary to avoid the one instruction ineterrupt disable window for sti when
2647 // FLAGS.IF is not set.
2648 "nop",
2649 "out 0x20, ax",
2650 "hlt",
2651 );
2652
2653 #[test]
test_interrupt_ready_when_interrupt_enable_flag_not_set()2654 fn test_interrupt_ready_when_interrupt_enable_flag_not_set() {
2655 let assembly = test_interrupt_ready_when_interrupt_enable_flag_not_set_code::data().to_vec();
2656 let setup = TestSetup {
2657 assembly: assembly.clone(),
2658 load_addr: GuestAddress(0x1000),
2659 initial_regs: Regs {
2660 rip: 0x1000,
2661 rflags: 2,
2662 ..Default::default()
2663 },
2664 ..Default::default()
2665 };
2666
2667 run_tests!(
2668 setup,
2669 |_, regs, _| {
2670 // For VMEXIT caused by HLT, the hypervisor will automatically advance the rIP register.
2671 assert_eq!(regs.rip, 0x1000 + assembly.len() as u64);
2672 },
2673 |_, exit, vcpu, _: &mut dyn Vm| {
2674 match exit {
2675 VcpuExit::Io => {
2676 let mut addr = 0;
2677 vcpu.handle_io(&mut |io_params| {
2678 addr = io_params.address;
2679 // We are always handling out IO port, so no data to return.
2680 })
2681 .expect("should handle IO successfully");
2682 let regs = vcpu
2683 .get_regs()
2684 .expect("should retrieve the registers successfully");
2685 match addr {
2686 0x10 => {
2687 assert_eq!(regs.rflags & FLAGS_IF_BIT, 0);
2688 assert!(!vcpu.ready_for_interrupt());
2689 }
2690 0x20 => {
2691 assert_eq!(regs.rflags & FLAGS_IF_BIT, FLAGS_IF_BIT);
2692 assert!(vcpu.ready_for_interrupt());
2693 }
2694 _ => panic!("unexpected addr: {}", addr),
2695 }
2696 false
2697 }
2698 VcpuExit::Hlt => true,
2699 r => panic!("unexpected VMEXIT reason: {:?}", r),
2700 }
2701 }
2702 );
2703 }
2704
2705 #[test]
test_enter_long_mode_direct()2706 fn test_enter_long_mode_direct() {
2707 global_asm_data!(
2708 pub long_mode_asm,
2709 ".code64",
2710 "mov rdx, rax",
2711 "mov rbx, [0x10000]",
2712 "hlt"
2713 );
2714
2715 let bigly_mem_value: u64 = 0x1_0000_0000;
2716 let biglier_mem_value: u64 = 0x1_0000_0001;
2717 let mut setup = TestSetup {
2718 assembly: long_mode_asm::data().to_vec(),
2719 mem_size: 0x11000,
2720 load_addr: GuestAddress(0x1000),
2721 initial_regs: Regs {
2722 rax: bigly_mem_value,
2723 rip: 0x1000,
2724 rflags: 0x2,
2725 ..Default::default()
2726 },
2727 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
2728 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
2729 })),
2730
2731 ..Default::default()
2732 };
2733
2734 setup.add_memory_initialization(
2735 GuestAddress(0x10000),
2736 biglier_mem_value.to_le_bytes().to_vec(),
2737 );
2738 let regs_matcher = move |_: HypervisorType, regs: &Regs, sregs: &Sregs| {
2739 assert!((sregs.efer & 0x400) != 0, "Long-Mode Active bit not set");
2740 assert_eq!(
2741 regs.rdx, bigly_mem_value,
2742 "Did not execute instructions correctly in long mode."
2743 );
2744 assert_eq!(
2745 regs.rbx, biglier_mem_value,
2746 "Was not able to access translated memory in long mode."
2747 );
2748 assert_eq!((sregs.cs.l), 1, "Long-mode bit not set in CS");
2749 };
2750
2751 let exit_matcher = |_, exit: &VcpuExit, _: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
2752 VcpuExit::Hlt => {
2753 true // Break VM runloop
2754 }
2755 r => panic!("unexpected exit reason: {:?}", r),
2756 };
2757
2758 run_tests!(setup, regs_matcher, exit_matcher);
2759 }
2760
2761 #[test]
test_enter_long_mode_asm()2762 fn test_enter_long_mode_asm() {
2763 global_asm_data!(
2764 pub enter_long_mode_asm,
2765 ".code16",
2766 "lidt [0xd100]", // Address of the IDT limit + base
2767 "mov eax, cr4",
2768 "or ax, 1 << 7 | 1 << 5", // Set the PAE-bit (bit 5) and PGE (bit 7).
2769 "mov cr4, eax",
2770
2771 "mov bx, 0x9000", // Address of the page table.
2772 "mov cr3, ebx",
2773
2774 "mov ecx, 0xC0000080", // Set ECX to EFER MSR (0xC0000080)
2775 "rdmsr", // Read from the MSR
2776 "or ax, 1 << 8", // Set the LM-bit (bit 8).
2777 "wrmsr", // Write to the MSR
2778
2779 "mov eax, cr0",
2780 "or eax, 1 << 31 | 1 << 0", // Set PG (31nd bit) & PM (0th bit).
2781 "mov cr0, eax",
2782
2783 "lgdt [0xd000]", // Address of the GDT limit + base
2784 "ljmp 16, 0xe000" // Address of long_mode_asm
2785 );
2786
2787 global_asm_data!(
2788 pub long_mode_asm,
2789 ".code64",
2790 "mov rdx, r8",
2791 "mov rbx, [0x10000]",
2792 "hlt"
2793 );
2794
2795 let bigly_mem_value: u64 = 0x1_0000_0000;
2796 let biglier_mem_value: u64 = 0x1_0000_0001;
2797 let mut setup = TestSetup {
2798 assembly: enter_long_mode_asm::data().to_vec(),
2799 mem_size: 0x13000,
2800 load_addr: GuestAddress(0x1000),
2801 initial_regs: Regs {
2802 r8: bigly_mem_value,
2803 rip: 0x1000,
2804 rflags: 0x2,
2805 ..Default::default()
2806 },
2807 extra_vm_setup: Some(Box::new(|_: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
2808 // TODO(b/354901961): configure_long_mode_memory loads GDT and IDT for 64 bit usage, and
2809 // the ABI doesn't match real mode and protected mode, but in this test, we first launch
2810 // in real mode.
2811
2812 ModeConfig::default_long_mode().configure_long_mode_memory(vm);
2813 })),
2814
2815 ..Default::default()
2816 };
2817
2818 setup.add_memory_initialization(
2819 GuestAddress(0x10000),
2820 biglier_mem_value.to_le_bytes().to_vec(),
2821 );
2822 setup.add_memory_initialization(GuestAddress(0xe000), long_mode_asm::data().to_vec());
2823
2824 // GDT limit + base, to be loaded by the lgdt instruction.
2825 // Must be within 0xFFFF as it's executed in real-mode.
2826 setup.add_memory_initialization(GuestAddress(0xd000), 0xFFFF_u32.to_le_bytes().to_vec());
2827 setup.add_memory_initialization(
2828 GuestAddress(0xd000 + 2),
2829 (DEFAULT_GDT_OFFSET as u32).to_le_bytes().to_vec(),
2830 );
2831
2832 // IDT limit + base, to be loaded by the lidt instruction.
2833 // Must be within 0xFFFF as it's executed in real-mode.
2834 setup.add_memory_initialization(GuestAddress(0xd100), 0xFFFF_u32.to_le_bytes().to_vec());
2835 setup.add_memory_initialization(
2836 GuestAddress(0xd100 + 2),
2837 (DEFAULT_IDT_OFFSET as u32).to_le_bytes().to_vec(),
2838 );
2839
2840 let regs_matcher = move |_: HypervisorType, regs: &Regs, sregs: &Sregs| {
2841 assert!((sregs.efer & 0x400) != 0, "Long-Mode Active bit not set");
2842 assert_eq!(
2843 regs.rdx, bigly_mem_value,
2844 "Did not execute instructions correctly in long mode."
2845 );
2846 assert_eq!(
2847 regs.rbx, biglier_mem_value,
2848 "Was not able to access translated memory in long mode."
2849 );
2850 assert_eq!((sregs.cs.l), 1, "Long-mode bit not set in CS");
2851 };
2852
2853 let exit_matcher = |_, exit: &VcpuExit, _: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
2854 VcpuExit::Hlt => {
2855 true // Break VM runloop
2856 }
2857 r => panic!("unexpected exit reason: {:?}", r),
2858 };
2859
2860 run_tests!(setup, regs_matcher, exit_matcher);
2861 }
2862
2863 #[test]
test_request_interrupt_window()2864 fn test_request_interrupt_window() {
2865 global_asm_data!(
2866 assembly,
2867 ".code16",
2868 // Disable the interrupt, and the interrupt window shouldn't cause a vcpu exit until the
2869 // interrupt is enabled again.
2870 "cli",
2871 // vcpu exit here to request an interrupt window when interrupt is not ready. We can't use
2872 // hlt for VMEXIT, because HAXM unconditionally allows interrupt injection for hlt.
2873 "out 0x10, ax",
2874 // Enable the interrupt.
2875 "sti",
2876 // Another instruction window for interrupt delivery after sti. We shouldn't receive the
2877 // interrupt window exit until we complete this instruction. We use another intercepted
2878 // instruction here to make sure the hypervisor doesn't shadow the not delivered interrupt
2879 // request window on an intercepted instruction.
2880 "out 0x10, ax",
2881 // WHPX requires another not intercepted instruction to restore from the not interruptible
2882 // state.
2883 "nop",
2884 // The interrupt window exit should happen either right before nop or right after nop.
2885 "hlt",
2886 );
2887
2888 let assembly = assembly::data().to_vec();
2889 let setup = TestSetup {
2890 assembly: assembly.clone(),
2891 load_addr: GuestAddress(0x1000),
2892 initial_regs: Regs {
2893 rip: 0x1000,
2894 rflags: 2,
2895 ..Default::default()
2896 },
2897 intercept_intr: true,
2898 ..Default::default()
2899 };
2900
2901 run_tests!(
2902 setup,
2903 |_, regs, _| assert_eq!(regs.rip, 0x1000 + assembly.len() as u64),
2904 {
2905 let mut io_counter = 0;
2906 let mut irq_window_received = false;
2907 move |hypervisor_type, exit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
2908 let is_irq_window = if hypervisor_type == HypervisorType::Haxm {
2909 matches!(exit, VcpuExit::Intr) && io_counter == 2
2910 } else {
2911 matches!(exit, VcpuExit::IrqWindowOpen)
2912 };
2913 if is_irq_window {
2914 assert_eq!(io_counter, 2);
2915 assert!(vcpu.ready_for_interrupt());
2916 vcpu.set_interrupt_window_requested(false);
2917
2918 irq_window_received = true;
2919 return false;
2920 }
2921 match exit {
2922 VcpuExit::Intr => false,
2923 VcpuExit::Io => {
2924 // We are always handling out IO port, so no data to return.
2925 vcpu.handle_io(&mut |_| {})
2926 .expect("should handle IO successfully");
2927
2928 assert!(!vcpu.ready_for_interrupt());
2929
2930 // Only set the interrupt window request on the first out instruction.
2931 if io_counter == 0 {
2932 vcpu.set_interrupt_window_requested(true);
2933 }
2934 io_counter += 1;
2935 false
2936 }
2937 VcpuExit::Hlt => {
2938 assert!(irq_window_received);
2939 true
2940 }
2941 r => panic!("unexpected VMEXIT: {:?}", r),
2942 }
2943 }
2944 }
2945 );
2946 }
2947
2948 #[test]
test_fsgsbase()2949 fn test_fsgsbase() {
2950 global_asm_data!(
2951 pub fsgsbase_asm,
2952 ".code64",
2953 "wrfsbase rax",
2954 "wrgsbase rbx",
2955 "rdfsbase rcx",
2956 "rdgsbase rdx",
2957 "mov rax, fs:0",
2958 "mov rbx, gs:0",
2959 "hlt"
2960 );
2961
2962 let code_addr = 0x1000;
2963 let fs = 0x10000;
2964 let gs = 0x10100;
2965
2966 let setup = TestSetup {
2967 assembly: fsgsbase_asm::data().to_vec(),
2968 mem_size: 0x11000,
2969 load_addr: GuestAddress(code_addr),
2970 initial_regs: Regs {
2971 rax: fs,
2972 rbx: gs,
2973 rip: code_addr,
2974 rflags: 0x2,
2975 ..Default::default()
2976 },
2977 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
2978 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
2979
2980 let mut sregs = vcpu.get_sregs().expect("unable to get sregs");
2981 sregs.cr4 |= 1 << 16; // FSGSBASE (bit 16)
2982 vcpu.set_sregs(&sregs).expect("unable to set sregs");
2983 })),
2984 memory_initializations: vec![
2985 (GuestAddress(fs), [0xaa; 8].into()),
2986 (GuestAddress(gs), [0xbb; 8].into()),
2987 ],
2988 ..Default::default()
2989 };
2990
2991 let regs_matcher = move |_: HypervisorType, regs: &Regs, sregs: &Sregs| {
2992 assert_eq!(regs.rcx, fs);
2993 assert_eq!(regs.rdx, gs);
2994 assert_eq!(regs.rax, 0xaaaaaaaaaaaaaaaa);
2995 assert_eq!(regs.rbx, 0xbbbbbbbbbbbbbbbb);
2996 assert_eq!(sregs.fs.base, fs);
2997 assert_eq!(sregs.gs.base, gs);
2998 };
2999
3000 let exit_matcher = |_, exit: &VcpuExit, _vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3001 VcpuExit::Hlt => {
3002 true // Break VM runloop
3003 }
3004 r => panic!("unexpected exit reason: {:?}", r),
3005 };
3006
3007 run_tests!(setup, regs_matcher, exit_matcher);
3008 }
3009
3010 /// Tests whether MMX state is being preserved by the hypervisor correctly (e.g. the hypervisor is
3011 /// properly using fxsave/fxrstor, or xsave/xrstor (or xsaves/xrstors)).
3012 #[test]
test_mmx_state_is_preserved_by_hypervisor()3013 fn test_mmx_state_is_preserved_by_hypervisor() {
3014 // This program stores a sentinel value into mm0 (the first MMX register) and verifies
3015 // that after a vmexit, that value is properly restored (we copy it to rbx so it can be checked
3016 // by the reg matcher when the VM hlts). In the vmexit handler function below, we make sure the
3017 // sentinel value is NOT in mm0. This way we know the mm0 value has changed, so we're guaranteed
3018 // the hypervisor has to restore the guest's sentinel value for the test to pass. (The read
3019 // from mm0 to rbx happens *after* the vmexit, so the hypervisor has to restore the guest's
3020 // mm0 otherwise there will be random garbage in there from the host. This would also be a
3021 // security issue.)
3022 //
3023 // Note: this program also verifies the guest has MMX support. If it does not, rdx will be 1 and
3024 // no MMX instructions will be attempted.
3025 let sentinel_mm0_value = 0x1337FFFFu64;
3026 global_asm_data!(
3027 pub mmx_ops_asm,
3028 ".code64",
3029 "mov eax, 1",
3030 "cpuid",
3031 "bt edx, 23",
3032 "jc HasMMX",
3033 "mov rdx, 1",
3034 "hlt",
3035 "HasMMX:",
3036 "xor rdx, rdx",
3037 "mov rax, 0x1337FFFF",
3038 "mov rbx, 0x0",
3039 "movq mm0, rax",
3040 "out 0x5, al",
3041 "movq rbx, mm0",
3042 "emms",
3043 "hlt",
3044 );
3045
3046 let code_addr = 0x1000;
3047 let setup = TestSetup {
3048 assembly: mmx_ops_asm::data().to_vec(),
3049 mem_size: 0x12000,
3050 load_addr: GuestAddress(code_addr),
3051 initial_regs: Regs {
3052 rip: code_addr,
3053 rflags: 0x2,
3054 ..Default::default()
3055 },
3056 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3057 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3058 })),
3059 memory_initializations: vec![],
3060 ..Default::default()
3061 };
3062
3063 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
3064 assert_ne!(regs.rdx, 1, "guest has no MMX support");
3065 assert_eq!(
3066 regs.rbx, sentinel_mm0_value,
3067 "guest MMX register not restored by hypervisor"
3068 );
3069 };
3070
3071 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3072 VcpuExit::Hlt => {
3073 true // Break VM runloop
3074 }
3075 VcpuExit::Cpuid { entry } => {
3076 vcpu.handle_cpuid(entry)
3077 .expect("should handle cpuid successfully");
3078 false
3079 }
3080 VcpuExit::Io => {
3081 vcpu.handle_io(&mut |_| {})
3082 .expect("should handle IO successfully");
3083
3084 // kaiyili@ pointed out we should check the XSAVE state exposed by the hypervisor via
3085 // its API (e.g. vm.get_xsave_state). This is used in snapshotting, so if it's wrong,
3086 // that would break things. It's also a good cross-check that the hypervisor is properly
3087 // handling xsave state.
3088 //
3089 // There are a couple of things blocking us from doing that today:
3090 // 1. gHAXM, our hypervisor of interest, doesn't expose its xsave area state for
3091 // the guest.
3092 // 2. We don't have an xsave area parser (yet).
3093
3094 // mm0 MUST NOT have the guest's sentinel value. If it somehow does, the hypervisor
3095 // didn't save the guest's FPU/MMX state / restore the host's state before exiting to
3096 // CrosVM.
3097 //
3098 // Note: MMX is ubiquitous on x86_64, so we don't check for support on the host (the
3099 // guest checks, so unless the guest's support is software implemented, it's highly
3100 // likely the host has MMX support).
3101 let mut mm0_value: u64;
3102 // SAFETY: we do not clobber any undeclared registers. Technically emms changes some
3103 // x87 state, so there's some UB risk here, but it is not explicitly called out by
3104 // the Rust docs as a bad idea.
3105 unsafe {
3106 asm!(
3107 "movq rax, mm0",
3108 "emms",
3109 out("rax") mm0_value);
3110 }
3111 assert_ne!(
3112 mm0_value, sentinel_mm0_value,
3113 "host mm0 value is the same as the guest sentinel value"
3114 );
3115 false
3116 }
3117 r => panic!("unexpected exit reason: {:?}", r),
3118 };
3119
3120 run_tests!(setup, regs_matcher, exit_matcher);
3121 }
3122
3123 /// Tests whether AVX state is being preserved by the hypervisor correctly (e.g. the hypervisor is
3124 /// properly using xsave/xrstor (or xsaves/xrstors)). This is very similar to the MMX test, but
3125 /// AVX state is *not* captured by fxsave, so that's how we guarantee xsave state of some kind is
3126 /// being handled properly.
3127 #[test]
test_avx_state_is_preserved_by_hypervisor()3128 fn test_avx_state_is_preserved_by_hypervisor() {
3129 if !is_x86_feature_detected!("avx") {
3130 panic!("this test requires host AVX support and it was not detected");
3131 }
3132
3133 let sentinel_value = 0x1337FFFFu64;
3134 global_asm_data!(
3135 pub avx_ops_asm,
3136 ".code64",
3137 "mov eax, 1",
3138 "cpuid",
3139 "bt ecx, 28",
3140 "jc HasAVX",
3141 "mov rdx, 1",
3142 "hlt",
3143 "HasAVX:",
3144
3145 // Turn on OSXSAVE (we can't touch XCR0 without it).
3146 "mov rax, cr4",
3147 "or eax, 1 << 18",
3148 "mov cr4, rax",
3149
3150 // AVX won't work unless we enable it.
3151 //
3152 // Set the relevant XCR0 bits:
3153 // 0: X87
3154 // 1: SSE
3155 // 2: AVX
3156 "xor rcx, rcx",
3157 "xgetbv",
3158 // (7 = 111b)
3159 "or eax, 7",
3160 "xsetbv",
3161
3162 // Now that AVX is ready to use, let's start with a clean slate (and signify we have AVX
3163 // support to the test assert below by zeroing rdx).
3164 "xor rdx, rdx",
3165 "xor rax, rax",
3166 "xor rbx, rbx",
3167 "vzeroall",
3168
3169 // Here's the actual test (finally). Since AVX is a little tricky to follow, here's what
3170 // the test does:
3171 // 1. We load 0x1337FFFF into ymm1 via xmm0.
3172 // 2. We perform port IO to exit out to CrosVM (our vmexit handler below).
3173 // 3. The vmexit handler makes sure ymm1 does NOT contain 0x1337FFFF.
3174 // 4. We return to this program. Then we dump the value of ymm1 into ebx. The exit
3175 // register matcher verifies that 0x1337FFFF is in ebx. This means the hypervisor
3176 // properly restored ymm1 for the guest on vmenter.
3177 "mov eax, 0x1337FFFF",
3178 "vpinsrd xmm0, xmm1, eax, 3",
3179 "vinserti128 ymm1, ymm2, xmm0, 1",
3180 "out 0x5, al",
3181 "vextracti128 xmm3, ymm1, 1",
3182 "vpextrd ebx, xmm3, 3",
3183 "hlt",
3184 );
3185
3186 let code_addr = 0x1000;
3187 let setup = TestSetup {
3188 assembly: avx_ops_asm::data().to_vec(),
3189 mem_size: 0x12000,
3190 load_addr: GuestAddress(code_addr),
3191 initial_regs: Regs {
3192 rip: code_addr,
3193 rflags: 0x2,
3194 ..Default::default()
3195 },
3196 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3197 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3198 })),
3199 memory_initializations: vec![],
3200 ..Default::default()
3201 };
3202
3203 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
3204 assert_ne!(regs.rdx, 1, "guest has no AVX support");
3205 assert_eq!(
3206 regs.rbx, sentinel_value,
3207 "guest AVX register not restored by hypervisor"
3208 );
3209 };
3210
3211 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3212 VcpuExit::Hlt => {
3213 true // Break VM runloop
3214 }
3215 VcpuExit::Cpuid { entry } => {
3216 vcpu.handle_cpuid(entry)
3217 .expect("should handle cpuid successfully");
3218 false
3219 }
3220 VcpuExit::Io => {
3221 vcpu.handle_io(&mut |_| {})
3222 .expect("should handle IO successfully");
3223
3224 // kaiyili@ pointed out we should check the XSAVE state exposed by the hypervisor via
3225 // its API (e.g. vm.get_xsave_state). This is used in snapshotting, so if it's wrong,
3226 // that would break things. It's also a good cross-check that the hypervisor is properly
3227 // handling xsave state.
3228 //
3229 // There are a couple of things blocking us from doing that today:
3230 // 1. gHAXM, our hypervisor of interest, doesn't expose its xsave area state for
3231 // the guest.
3232 // 2. We don't have a xsave area parser (yet).
3233
3234 // ymm1 MUST NOT have the guest's sentinel value. If it somehow does, the hypervisor
3235 // didn't save the guest's AVX state / restore the host's state before exiting to
3236 // CrosVM.
3237 //
3238 // Note: AVX is ubiquitous on x86_64, so we don't check for support on the host (the
3239 // guest checks, so unless the guest's support is software implemented, it's highly
3240 // likely the host has AVX support).
3241 let mut ymm1_sub_value: u64;
3242 // SAFETY: we don't clobber any undeclared registers.
3243 unsafe {
3244 asm!(
3245 "vextracti128 xmm4, ymm1, 1",
3246 "vpextrd eax, xmm4, 3",
3247 out("rax") ymm1_sub_value,
3248 out("xmm4") _);
3249 }
3250 assert_ne!(
3251 ymm1_sub_value, sentinel_value,
3252 "host ymm1 value is the same as the guest sentinel value. Hypervisor likely didn't \
3253 save guest's state."
3254 );
3255 false
3256 }
3257 r => panic!("unexpected exit reason: {:?}", r),
3258 };
3259
3260 run_tests!(setup, regs_matcher, exit_matcher);
3261 }
3262
3263 /// Tests whether XSAVE works inside a guest.
3264 #[test]
test_xsave()3265 fn test_xsave() {
3266 let sentinel_xmm0_value = 0x1337FFFFu64;
3267 global_asm_data!(
3268 pub xsave_ops_asm,
3269 ".code64",
3270
3271 // Make sure XSAVE is supported.
3272 "mov eax, 1",
3273 "mov ecx, 0",
3274 "cpuid",
3275 "bt ecx, 26",
3276 "jc HasXSAVE",
3277 "mov rdx, 1",
3278 "hlt",
3279 "HasXSAVE:",
3280 "xor rdx, rdx",
3281
3282 // Turn on OSXSAVE.
3283 "mov rax, cr4",
3284 "or eax, 1 << 18",
3285 "mov cr4, rax",
3286
3287 // Enable X87, SSE, and AVX.
3288 //
3289 // Set the relevant XCR0 bits:
3290 // 0: X87
3291 // 1: SSE
3292 // 3: AVX
3293 "xor rcx, rcx",
3294 "xgetbv",
3295 // (7 = 111b)
3296 "or eax, 7",
3297 "xsetbv",
3298
3299 // Put the sentinel value in xmm0, and save it off.
3300 "mov eax, 0x1337FFFF",
3301 "vzeroall",
3302 "vpinsrd xmm0, xmm1, eax, 3",
3303 "xor edx, edx",
3304 "mov eax, 7",
3305 "xsave dword ptr [0x10000]",
3306
3307 // Clear xmm0.
3308 "vpxor xmm0, xmm0, xmm0",
3309
3310 // Restoring should put the sentinel value back.
3311 "xor edx, edx",
3312 "mov eax, 7",
3313 "xrstor dword ptr [0x10000]",
3314
3315 "xor rbx, rbx",
3316 "vpextrd ebx, xmm0, 3",
3317 "hlt",
3318 );
3319
3320 let code_addr = 0x1000;
3321 let setup = TestSetup {
3322 assembly: xsave_ops_asm::data().to_vec(),
3323 mem_size: 0x12000,
3324 load_addr: GuestAddress(code_addr),
3325 initial_regs: Regs {
3326 rip: code_addr,
3327 rflags: 0x2,
3328 ..Default::default()
3329 },
3330 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3331 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3332 })),
3333 memory_initializations: vec![(GuestAddress(0x10000), vec![0; 0x1000])],
3334 ..Default::default()
3335 };
3336
3337 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
3338 assert_ne!(regs.rdx, 1, "guest has no XSAVE support");
3339 assert_eq!(
3340 regs.rbx, sentinel_xmm0_value,
3341 "guest SSE register not restored by XRSTOR",
3342 );
3343 };
3344
3345 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3346 VcpuExit::Hlt => {
3347 true // Break VM runloop
3348 }
3349 VcpuExit::Cpuid { entry } => {
3350 vcpu.handle_cpuid(entry)
3351 .expect("should handle cpuid successfully");
3352 false
3353 }
3354 VcpuExit::MsrAccess => false, // MsrAccess handled by hypervisor impl
3355 r => panic!("unexpected exit reason: {:?}", r),
3356 };
3357
3358 run_tests!(setup, regs_matcher, exit_matcher);
3359 }
3360
3361 /// Tests whether XSAVES works inside a guest.
3362 ///
3363 /// Ignored because CET is not available in some nested virtualization
3364 /// environments (such as CI). (CET is the feature we use to test XSAVES.)
3365 #[ignore]
3366 #[cfg(feature = "whpx")]
3367 #[test]
test_xsaves()3368 fn test_xsaves() {
3369 global_asm_data!(
3370 pub xsaves_ops_asm,
3371 ".code64",
3372
3373 // Make sure XSAVES is supported.
3374 "mov eax, 0xd",
3375 "mov ecx, 1",
3376 "cpuid",
3377 "bt eax, 3",
3378 "jc HasXSAVES",
3379 "mov rdx, 1",
3380 "hlt",
3381 "HasXSAVES:",
3382
3383 // Make sure CET is supported.
3384 "mov eax, 7",
3385 "mov ecx, 0",
3386 "cpuid",
3387 "bt ecx, 7",
3388 "jc HasCET",
3389 "mov rdx, 2",
3390 "hlt",
3391 "HasCET:",
3392
3393 // Turn on write protection for ring 0 (required by CET).
3394 "mov rax, cr0",
3395 "or eax, 1 << 16",
3396 "mov cr0, rax",
3397
3398 // Turn on OSXSAVE (18) and CET (23).
3399 "mov rax, cr4",
3400 "or eax, 1 << 18",
3401 "or eax, 1 << 23",
3402 "mov cr4, rax",
3403
3404 // Set up XSAVES to manage CET state.
3405 // IA32_XSS = 0x0DA0
3406 "mov ecx, 0x0DA0",
3407 "rdmsr",
3408 "or eax, 1 << 12",
3409 "wrmsr",
3410
3411 // Enable CET.
3412 "mov ecx, 0x6A2",
3413 "rdmsr",
3414 "or eax, 1",
3415 "wrmsr",
3416
3417 // Now CET is usable and managed by XSAVES. Let's set a sentinel value and make sure xsaves
3418 // restores it as expected. Note that PL0_SSP's linear address must be 8 byte aligned.
3419 // PL0_SSP = 0x06A5
3420 "mov ecx, 0x06A4",
3421 "xor edx, edx",
3422 "xor eax, eax",
3423 "mov eax, 0x13370000",
3424 "wrmsr",
3425
3426 // Set the RFBM / feature mask to include CET.
3427 "xor edx, edx",
3428 "mov eax, 1 << 12",
3429 "xsaves dword ptr [0x10000]",
3430
3431 // Clear PL0_SSP
3432 "xor edx, edx",
3433 "xor eax, eax",
3434 "mov ecx, 0x06A4",
3435 "wrmsr",
3436
3437 // Set the RFBM / feature mask to include CET.
3438 "xor edx, edx",
3439 "mov eax, 1 << 12",
3440 "xrstors dword ptr [0x10000]",
3441
3442 // Check to see if PL0_SSP was restored.
3443 "mov ecx, 0x06A4",
3444 "rdmsr",
3445 "cmp eax, 0x13370000",
3446 "jz TestPasses",
3447 "mov rdx, 3",
3448 "hlt",
3449 "TestPasses:",
3450 "xor rdx, rdx",
3451 "hlt",
3452 );
3453
3454 let code_addr = 0x1000;
3455 let setup = TestSetup {
3456 assembly: xsaves_ops_asm::data().to_vec(),
3457 mem_size: 0x12000,
3458 load_addr: GuestAddress(code_addr),
3459 initial_regs: Regs {
3460 rip: code_addr,
3461 rdx: 0x4,
3462 rflags: 0x2,
3463 ..Default::default()
3464 },
3465 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3466 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3467 })),
3468 memory_initializations: vec![(GuestAddress(0x10000), vec![0; 0x1000])],
3469 ..Default::default()
3470 };
3471
3472 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
3473 assert_ne!(regs.rdx, 1, "guest has no XSAVES support");
3474 assert_ne!(regs.rdx, 2, "guest has no CET support");
3475 assert_ne!(regs.rdx, 3, "guest didn't restore PL0_SSP as expected");
3476 assert_eq!(regs.rdx, 0, "test failed unexpectedly");
3477 };
3478
3479 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3480 VcpuExit::Hlt => {
3481 true // Break VM runloop
3482 }
3483 VcpuExit::Cpuid { entry } => {
3484 vcpu.handle_cpuid(entry)
3485 .expect("should handle cpuid successfully");
3486 false
3487 }
3488 VcpuExit::MsrAccess => false, // MsrAccess handled by hypervisor impl
3489 r => panic!("unexpected exit reason: {:?}", r),
3490 };
3491
3492 run_tests!(setup, regs_matcher, exit_matcher);
3493 }
3494
3495 /// Tests that XSAVES is disabled in gHAXM (it's unsupported).
3496 ///
3497 /// Note: this test passing in CI is not necessarily a signal that gHAXM is working correctly
3498 /// because XSAVES is disabled in some nested virtualization environments (e.g. CI).
3499 #[cfg(feature = "haxm")]
3500 #[test]
test_xsaves_is_disabled_on_haxm()3501 fn test_xsaves_is_disabled_on_haxm() {
3502 global_asm_data!(
3503 pub no_xsaves_asm,
3504 ".code64",
3505
3506 "mov eax, 0xd",
3507 "mov ecx, 1",
3508 "cpuid",
3509 "bt eax, 3",
3510 "jnc NoXSAVES",
3511 "mov rdx, 1",
3512 "hlt",
3513 "NoXSAVES:",
3514 "mov rdx, 0",
3515 "hlt",
3516 );
3517
3518 let code_addr = 0x1000;
3519 let setup = TestSetup {
3520 assembly: no_xsaves_asm::data().to_vec(),
3521 mem_size: 0x12000,
3522 load_addr: GuestAddress(code_addr),
3523 initial_regs: Regs {
3524 rip: code_addr,
3525 rdx: 0x2,
3526 rflags: 0x2,
3527 ..Default::default()
3528 },
3529 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3530 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3531 })),
3532 memory_initializations: vec![],
3533 ..Default::default()
3534 };
3535
3536 let regs_matcher = move |_: HypervisorType, regs: &Regs, _: &_| {
3537 assert_ne!(regs.rdx, 1, "guest has XSAVES support and shouldn't");
3538 assert_eq!(regs.rdx, 0, "test failed unexpectedly");
3539 };
3540
3541 let exit_matcher = |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3542 VcpuExit::Hlt => {
3543 true // Break VM runloop
3544 }
3545 VcpuExit::Cpuid { entry } => {
3546 vcpu.handle_cpuid(entry)
3547 .expect("should handle cpuid successfully");
3548 false
3549 }
3550 VcpuExit::MsrAccess => false, // MsrAccess handled by hypervisor impl
3551 r => panic!("unexpected exit reason: {:?}", r),
3552 };
3553
3554 run_tests!(setup, regs_matcher, exit_matcher);
3555 }
3556
3557 /// Tests whether SLAT is updated properly when a region is removed from the guest. A correctly
3558 /// implemented hypervisor will flush the TLB such that this immediately hits a SLAT fault and comes
3559 /// to us as MMIO. If we don't see that, and the guest actually reads from the removed region, the
3560 /// test will fail. In the real world, this would be a guest read from a random pfn, which is
3561 /// UB (and a major security problem).
3562 ///
3563 /// Flakes should be treated as real failures (this test can show a false negative, but never a
3564 /// false positive).
3565 #[test]
test_slat_on_region_removal_is_mmio()3566 fn test_slat_on_region_removal_is_mmio() {
3567 global_asm_data!(
3568 pub test_asm,
3569 ".code64",
3570
3571 // Load the TLB with a mapping for the test region.
3572 "mov al, byte ptr [0x20000]",
3573
3574 // Signal to the host that VM is running. On this vmexit, the host will unmap the test
3575 // region.
3576 "out 0x5, al",
3577
3578 // This read should result in MMIO, and if it does, the test passes. If we hit the hlt, then
3579 // the test fails (since it means we were able to satisfy this read without exiting).
3580 "mov al, byte ptr [0x20000]",
3581 "hlt"
3582 );
3583
3584 const TEST_MEM_REGION_SIZE: usize = 0x1000;
3585 let memslot: Arc<Mutex<Option<MemSlot>>> = Arc::new(Mutex::new(None));
3586 let memslot_for_func = memslot.clone();
3587
3588 let code_addr = 0x1000;
3589 let setup = TestSetup {
3590 assembly: test_asm::data().to_vec(),
3591 mem_size: 0x12000,
3592 load_addr: GuestAddress(code_addr),
3593 initial_regs: Regs {
3594 rip: code_addr,
3595 rflags: 0x2,
3596 ..Default::default()
3597 },
3598 extra_vm_setup: Some(Box::new(
3599 move |vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
3600 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
3601
3602 // Create a test pinned memory region that is all 0xFF.
3603 let shm = SharedMemory::new("test", TEST_MEM_REGION_SIZE as u64).unwrap();
3604 let test_region = Box::new(
3605 MemoryMappingBuilder::new(TEST_MEM_REGION_SIZE)
3606 .from_shared_memory(&shm)
3607 .build()
3608 .unwrap(),
3609 );
3610 let ff_init = [0xFFu8; TEST_MEM_REGION_SIZE];
3611 test_region.write_slice(&ff_init, 0).unwrap();
3612 let test_region = Box::new(
3613 PinnedMemoryRegion::new(test_region).expect("failed to pin test region"),
3614 );
3615 *memslot_for_func.lock() = Some(
3616 vm.add_memory_region(
3617 GuestAddress(0x20000),
3618 test_region,
3619 false,
3620 false,
3621 MemCacheType::CacheCoherent,
3622 )
3623 .unwrap(),
3624 );
3625 },
3626 )),
3627 memory_initializations: vec![],
3628 ..Default::default()
3629 };
3630
3631 // Holds the test memory region after it's unmapped and the VM is still running. Without this,
3632 // incorrect access to the region by the VM would be unsafe / UB.
3633 let test_region_arc: Arc<Mutex<Option<Box<dyn MappedRegion>>>> = Arc::new(Mutex::new(None));
3634 let test_region_arc_for_exit = test_region_arc.clone();
3635
3636 let exit_matcher =
3637 move |_, exit: &VcpuExit, vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| match exit {
3638 VcpuExit::Io => {
3639 // WHPX insists on data being returned here or it throws MemoryCallbackFailed.
3640 //
3641 // We strictly don't care what this data is, since the VM exits before running any
3642 // further instructions.
3643 vcpu.handle_io(&mut |_| {})
3644 .expect("should handle IO successfully");
3645
3646 // Remove the test memory region to cause a SLAT fault (in the passing case).
3647 //
3648 // This also ensures the memory region remains pinned in host physical memory so any
3649 // incorrect accesses to it by the VM will remain safe.
3650 *test_region_arc_for_exit.lock() =
3651 Some(vm.remove_memory_region(memslot.lock().unwrap()).unwrap());
3652 false
3653 }
3654 VcpuExit::Mmio => {
3655 vcpu.handle_mmio(&mut |IoParams { address, operation }| {
3656 assert_eq!(address, 0x20000, "MMIO for wrong address");
3657 match operation {
3658 IoOperation::Read(data) => {
3659 assert_eq!(data.len(), 1);
3660 data[0] = 0;
3661 Ok(())
3662 }
3663 IoOperation::Write(_) => {
3664 panic!("got unexpected IO operation {:?}", operation);
3665 }
3666 }
3667 })
3668 .unwrap();
3669 true
3670 }
3671 VcpuExit::Hlt => {
3672 panic!("VM should not reach the hlt instruction (MMIO should've ended the VM)");
3673 }
3674 r => panic!("unexpected exit reason: {:?}", r),
3675 };
3676
3677 // We want to catch if the hypervisor doesn't clear the VM's TLB. If we hop between CPUs, then
3678 // we're likely to end up with a clean TLB on another CPU.
3679 set_cpu_affinity(vec![0]).unwrap();
3680
3681 run_tests!(setup, move |_, _, _| {}, &exit_matcher);
3682 }
3683
3684 struct PinnedMemoryRegion {
3685 mem_region: Box<dyn MappedRegion>,
3686 }
3687
3688 impl PinnedMemoryRegion {
new(mem_region: Box<dyn MappedRegion>) -> base::Result<Self>3689 fn new(mem_region: Box<dyn MappedRegion>) -> base::Result<Self> {
3690 // SAFETY:
3691 // ptr is a valid pointer and points to a region of the supplied size.
3692 unsafe { pin_memory(mem_region.as_ptr() as *mut _, mem_region.size()) }?;
3693 Ok(Self { mem_region })
3694 }
3695 }
3696
3697 // SAFETY:
3698 // Safe because ptr & size a memory range owned by this MemoryMapping that won't be unmapped
3699 // until it's dropped.
3700 unsafe impl MappedRegion for PinnedMemoryRegion {
as_ptr(&self) -> *mut u83701 fn as_ptr(&self) -> *mut u8 {
3702 self.mem_region.as_ptr()
3703 }
3704
size(&self) -> usize3705 fn size(&self) -> usize {
3706 self.mem_region.size()
3707 }
3708 }
3709
3710 impl Drop for PinnedMemoryRegion {
drop(&mut self)3711 fn drop(&mut self) {
3712 // SAFETY:
3713 // memory region passed is a valid pointer and points to a region of the
3714 // supplied size. We also panic on failure.
3715 unsafe { unpin_memory(self.mem_region.as_ptr() as *mut _, self.mem_region.size()) }
3716 .expect("failed to unpin memory")
3717 }
3718 }
3719
pin_memory(ptr: *mut c_void, len: usize) -> base::Result<()>3720 unsafe fn pin_memory(ptr: *mut c_void, len: usize) -> base::Result<()> {
3721 #[cfg(windows)]
3722 {
3723 if VirtualLock(ptr, len).into() {
3724 Ok(())
3725 } else {
3726 Err(base::Error::last())
3727 }
3728 }
3729 #[cfg(unix)]
3730 {
3731 if libc::mlock(ptr, len) != 0 {
3732 Err(base::Error::last())
3733 } else {
3734 Ok(())
3735 }
3736 }
3737 }
3738
unpin_memory(ptr: *mut c_void, len: usize) -> base::Result<()>3739 unsafe fn unpin_memory(ptr: *mut c_void, len: usize) -> base::Result<()> {
3740 #[cfg(windows)]
3741 {
3742 if VirtualUnlock(ptr, len).into() {
3743 Ok(())
3744 } else {
3745 Err(base::Error::last())
3746 }
3747 }
3748 #[cfg(unix)]
3749 {
3750 if libc::munlock(ptr, len) != 0 {
3751 Err(base::Error::last())
3752 } else {
3753 Ok(())
3754 }
3755 }
3756 }
3757
3758 #[test]
test_interrupt_injection_when_not_ready()3759 fn test_interrupt_injection_when_not_ready() {
3760 // This test ensures that if we inject an interrupt when it's not ready for interrupt, we
3761 // shouldn't end up with crash or hang. And if the interrupt is delivered, it shouldn't be
3762 // delivered before we reenable the interrupt.
3763 mod assembly {
3764 use super::*;
3765
3766 global_asm_data!(
3767 pub init,
3768 ".code16",
3769 // Set the IDT
3770 "lidt [0x200]",
3771 // Set up the stack, which will be used when CPU transfers the control to the ISR on
3772 // interrupt.
3773 "mov sp, 0x900",
3774 // Set ax to 0.
3775 "xor ax, ax",
3776 // Set the address 0x910 to 1 when we disable the interrupt, and restore it to 0 after
3777 // we renable the interrupt.
3778 "mov word ptr [0x910], 1",
3779 "cli",
3780 // We can't use hlt for VMEXIT, because HAXM unconditionally allows interrupt injection
3781 // for hlt. We will inject an interrupt here although all hypervisors should report not
3782 // ready for injection an interrupt. And we don't care if the injection succeeds or not.
3783 "out 0x10, ax",
3784 "sti",
3785 // Set the address 0x910 to 0 when we renable the interrupt.
3786 "mov word ptr [0x910], 0",
3787 // For hypervisor that injects the interrupt later when it's ready, the interrupt will
3788 // be delivered here.
3789 "nop",
3790 "hlt",
3791 );
3792
3793 // We still need an ISR in case the hypervisor actually delivers an interrupt.
3794 global_asm_data!(
3795 pub isr,
3796 ".code16",
3797 // ax will be 0 if the interrupt is delivered after we reenable the interrupt.
3798 // Otherwise, ax will be 1, and the test fails.
3799 "mov ax, word ptr [0x910]",
3800 "iret",
3801 );
3802 }
3803
3804 let start_addr: u32 = 0x200;
3805 // Allocate exceed 0x900, where we set up our stack.
3806 let mem_size: u32 = 0x1000;
3807
3808 let mut setup = TestSetup {
3809 load_addr: GuestAddress(start_addr.into()),
3810 initial_regs: Regs {
3811 rax: 0,
3812 // Set RFLAGS.IF to enable interrupt at the beginning.
3813 rflags: 2 | FLAGS_IF_BIT,
3814 ..Default::default()
3815 },
3816 mem_size: mem_size.into(),
3817 ..Default::default()
3818 };
3819
3820 let mut cur_addr = start_addr;
3821
3822 let idtr_size: u32 = 6;
3823 assert_eq!(
3824 Ok(std::mem::size_of::<Idtr32>()),
3825 usize::try_from(idtr_size)
3826 );
3827 // The limit is calculated from 256 entries timed by 4 bytes per entry.
3828 let idt_size = 256u16 * 4u16;
3829 let idtr = Idtr32 {
3830 limit: idt_size - 1,
3831 // The IDT right follows the IDTR.
3832 base_address: start_addr + idtr_size,
3833 };
3834 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idtr.as_bytes().to_vec());
3835 cur_addr += idtr_size;
3836
3837 let idt_entry = (start_addr + idtr_size + u32::from(idt_size)).to_ne_bytes();
3838 // IDT entries are far pointers(CS:IP pair) to the only ISR, which locates right after the IDT.
3839 // We set all entries to the same ISR.
3840 let idt = (0..256).flat_map(|_| idt_entry).collect::<Vec<_>>();
3841 setup.add_memory_initialization(GuestAddress(cur_addr.into()), idt.clone());
3842 cur_addr += u32::try_from(idt.len()).expect("IDT size should be within u32");
3843
3844 let isr_assembly = assembly::isr::data().to_vec();
3845 setup.add_memory_initialization(GuestAddress(cur_addr.into()), isr_assembly.clone());
3846 cur_addr += u32::try_from(isr_assembly.len()).expect("ISR size should be within u32");
3847
3848 let init_assembly = assembly::init::data().to_vec();
3849 setup.initial_regs.rip = cur_addr.into();
3850 setup.add_memory_initialization(GuestAddress(cur_addr.into()), init_assembly.clone());
3851 cur_addr += u32::try_from(init_assembly.len()).expect("init size should be within u32");
3852
3853 assert!(mem_size > cur_addr);
3854
3855 run_tests!(
3856 setup,
3857 |_, regs, _| {
3858 assert_eq!(
3859 regs.rax, 0,
3860 "the interrupt should be either not delivered(ax is kept as the initial value 0) \
3861 or is delivered after we reenable the interrupt(when the ax is set from 0x910, \
3862 0x910 is 0)"
3863 );
3864 },
3865 |_, exit, vcpu: &mut dyn VcpuX86_64, _: &mut dyn Vm| {
3866 match exit {
3867 // We exit and pass the test either the VCPU run fails or we hit hlt.
3868 VcpuExit::FailEntry { .. } | VcpuExit::Shutdown(..) | VcpuExit::Hlt => true,
3869 VcpuExit::Io => {
3870 // We are always handling out IO port, so no data to return.
3871 vcpu.handle_io(&mut |_| {})
3872 .expect("should handle IO successfully");
3873 assert!(!vcpu.ready_for_interrupt());
3874 // We don't care whether we inject the interrupt successfully or not.
3875 let _ = vcpu.interrupt(32);
3876 false
3877 }
3878 r => panic!("unexpected VMEXIT reason: {:?}", r),
3879 }
3880 }
3881 );
3882 }
3883
3884 #[test]
test_ready_for_interrupt_for_intercepted_instructions()3885 fn test_ready_for_interrupt_for_intercepted_instructions() {
3886 global_asm_data!(
3887 assembly,
3888 // We will use out instruction to cause VMEXITs and test ready_for_interrupt then.
3889 ".code16",
3890 // Disable the interrupt.
3891 "cli",
3892 // ready_for_interrupt should be false here.
3893 "out 0x10, ax",
3894 "sti",
3895 // ready_for_interrupt should be false here, because of the one instruction
3896 // interruptibility window for sti. And this is also an intercepted instruction.
3897 "out 0x20, ax",
3898 // ready_for_interrupt should be true here except for WHPX.
3899 "out 0x30, ax",
3900 // Restore the interruptibility for WHPX.
3901 "nop",
3902 "mov ax, ss",
3903 "mov ss, ax",
3904 // ready_for_interrupt should be false here, because of the one instruction
3905 // interruptibility window for mov ss. And this is also an intercepted instruction.
3906 "out 0x40, ax",
3907 // ready_for_interrupt should be true here except for WHPX.
3908 "out 0x50, ax",
3909 "hlt"
3910 );
3911
3912 let assembly = assembly::data().to_vec();
3913 let setup = TestSetup {
3914 assembly: assembly.clone(),
3915 load_addr: GuestAddress(0x1000),
3916 initial_regs: Regs {
3917 rip: 0x1000,
3918 rflags: 2,
3919 ..Default::default()
3920 },
3921 ..Default::default()
3922 };
3923
3924 run_tests!(
3925 setup,
3926 |_, regs, _| {
3927 // For VMEXIT caused by HLT, the hypervisor will automatically advance the rIP register.
3928 assert_eq!(regs.rip, 0x1000 + assembly.len() as u64);
3929 },
3930 |hypervisor_type, exit, vcpu, _: &mut dyn Vm| {
3931 match exit {
3932 VcpuExit::Hlt => true,
3933 VcpuExit::Io => {
3934 let ready_for_interrupt = vcpu.ready_for_interrupt();
3935 let mut io_port = 0;
3936 vcpu.handle_io(&mut |params| {
3937 io_port = params.address;
3938 // We are always handling out IO port, so no data to return.
3939 })
3940 .expect("should handle port IO successfully");
3941 match io_port {
3942 0x10 | 0x20 | 0x40 => assert!(!ready_for_interrupt),
3943 0x30 | 0x50 => {
3944 // WHPX needs a not intercepted instruction to recover to the proper
3945 // interruptibility state.
3946 if hypervisor_type != HypervisorType::Whpx {
3947 assert!(ready_for_interrupt);
3948 }
3949 }
3950 _ => panic!("unexpected port {}", io_port),
3951 }
3952 false
3953 }
3954 r => panic!("unexpected exit reason: {:?}", r),
3955 }
3956 }
3957 );
3958 }
3959
3960 #[cfg(feature = "haxm")]
3961 #[test]
test_cpuid_mwait_not_supported()3962 fn test_cpuid_mwait_not_supported() {
3963 global_asm_data!(
3964 cpuid_code,
3965 ".code64",
3966 "mov eax, 1", // CPUID function 1
3967 "cpuid",
3968 "hlt"
3969 );
3970
3971 let setup = TestSetup {
3972 assembly: cpuid_code::data().to_vec(),
3973 load_addr: GuestAddress(0x1000),
3974 initial_regs: Regs {
3975 rip: 0x1000,
3976 rflags: 2,
3977 ..Default::default()
3978 },
3979 ..Default::default()
3980 };
3981
3982 let regs_matcher = |_: HypervisorType, regs: &Regs, _: &Sregs| {
3983 // Check if MWAIT is not supported
3984 assert_eq!(
3985 regs.rcx & (1 << 3),
3986 0,
3987 "MWAIT is supported, but it should not be."
3988 );
3989 };
3990
3991 let exit_matcher = |_, exit: &VcpuExit, _: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
3992 VcpuExit::Hlt => {
3993 true // Break VM runloop
3994 }
3995 r => panic!("unexpected exit reason: {:?}", r),
3996 };
3997
3998 run_tests!(setup, regs_matcher, exit_matcher);
3999 }
4000
4001 #[test]
test_hardware_breakpoint_with_isr()4002 fn test_hardware_breakpoint_with_isr() {
4003 global_asm_data!(
4004 setup_debug_handler_code,
4005 ".code64",
4006 // Set up the stack
4007 "mov sp, 0x900",
4008 "mov rax, 0x1019", // Address of the instruction to trigger the breakpoint
4009 "mov dr0, rax",
4010 "mov rax, 0x00000001", // Enable the first breakpoint (local, exact) for execution
4011 "mov dr7, rax",
4012 "nop", // This should trigger the debug exception
4013 "nop",
4014 "hlt"
4015 );
4016
4017 global_asm_data!(
4018 debug_isr_code,
4019 ".code64",
4020 "mov rbx, 0xf00dbabe", // Set a value to indicate the ISR was called
4021 "mov rax, 0",
4022 "mov dr7, rax", // Disable debugging again
4023 "mov rax, dr6",
4024 "iretq" // Return from interrupt
4025 );
4026
4027 global_asm_data!(
4028 null_isr_code,
4029 ".code64",
4030 "mov rbx, 0xbaadf00d", // This ISR should never get called
4031 "hlt"
4032 );
4033
4034 let debug_isr_offset = 0x800;
4035 let null_isr_offset = 0x700;
4036 let debug_idt_entry = IdtEntry64::new(debug_isr_offset);
4037 let null_idt_entry = IdtEntry64::new(null_isr_offset);
4038
4039 let setup = TestSetup {
4040 assembly: setup_debug_handler_code::data().to_vec(),
4041 load_addr: GuestAddress(0x1000),
4042 mem_size: 0x20000,
4043 initial_regs: Regs {
4044 rip: 0x1000,
4045 rflags: 2 | FLAGS_IF_BIT,
4046 ..Default::default()
4047 },
4048 extra_vm_setup: Some(Box::new(
4049 move |vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
4050 let guest_mem = vm.get_memory();
4051
4052 guest_mem
4053 .write_at_addr(
4054 debug_isr_code::data().to_vec().as_bytes(),
4055 GuestAddress(debug_isr_offset),
4056 )
4057 .expect("Failed to write debug ISR entry");
4058
4059 guest_mem
4060 .write_at_addr(
4061 null_isr_code::data().to_vec().as_bytes(),
4062 GuestAddress(null_isr_offset),
4063 )
4064 .expect("Failed to write null ISR entry");
4065
4066 let mut long_mode_config = ModeConfig::default_long_mode();
4067 long_mode_config
4068 .set_idt_long_mode((0..256).map(|i| {
4069 if i == 0x01 {
4070 debug_idt_entry
4071 } else {
4072 null_idt_entry
4073 }
4074 }))
4075 .set_idt_base_addr(0x12_000);
4076 long_mode_config.enter_long_mode(vcpu, vm);
4077 },
4078 )),
4079 ..Default::default()
4080 };
4081
4082 let regs_matcher = |_: HypervisorType, regs: &Regs, _: &Sregs| {
4083 assert_eq!(regs.rax & 1, 1, "Breakpoint #0 not hit");
4084 assert_eq!(
4085 regs.rip,
4086 0x1000 + (setup_debug_handler_code::data().len() as u64),
4087 "rIP not at the right HLT"
4088 );
4089 assert_eq!(regs.rbx, 0xf00dbabe, "Debug ISR was not called");
4090 };
4091
4092 let exit_matcher = |_, exit: &VcpuExit, _: &mut dyn VcpuX86_64, _: &mut dyn Vm| match exit {
4093 VcpuExit::Hlt => {
4094 true // Break VM runloop
4095 }
4096 r => panic!("unexpected exit reason: {:?}", r),
4097 };
4098
4099 run_tests!(setup, regs_matcher, exit_matcher);
4100 }
4101
4102 #[test]
test_debug_register_persistence()4103 fn test_debug_register_persistence() {
4104 global_asm_data!(
4105 test_debug_registers_code,
4106 ".code64",
4107 "mov dr0, rax",
4108 "inc rax",
4109 "mov dr1, rax",
4110 "inc rax",
4111 "mov dr2, rax",
4112 "inc rax",
4113 "mov dr3, rax",
4114 // Perform HLT to cause VMEXIT
4115 "hlt",
4116 "mov r8, dr0",
4117 "mov r9, dr1",
4118 "mov r10, dr2",
4119 "mov r11, dr3",
4120 "hlt"
4121 );
4122
4123 let initial_dr_value: u64 = 0x12345678;
4124
4125 let setup = TestSetup {
4126 assembly: test_debug_registers_code::data().to_vec(),
4127 mem_size: 0x11000,
4128 load_addr: GuestAddress(0x1000),
4129 initial_regs: Regs {
4130 rax: initial_dr_value,
4131 rip: 0x1000,
4132 rflags: 2,
4133 ..Default::default()
4134 },
4135 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
4136 ModeConfig::default_long_mode().enter_long_mode(vcpu, vm);
4137 })),
4138 ..Default::default()
4139 };
4140
4141 let mut hlt_count = 0;
4142
4143 run_tests!(
4144 setup,
4145 |_, regs, _| {
4146 assert_eq!(regs.r8, initial_dr_value, "DR0 value mismatch after VMEXIT");
4147 assert_eq!(
4148 regs.r9,
4149 initial_dr_value + 1,
4150 "DR1 value mismatch after VMEXIT"
4151 );
4152 assert_eq!(
4153 regs.r10,
4154 initial_dr_value + 2,
4155 "DR2 value mismatch after VMEXIT"
4156 );
4157 assert_eq!(
4158 regs.r11,
4159 initial_dr_value + 3,
4160 "DR3 value mismatch after VMEXIT"
4161 );
4162 },
4163 |_, exit, _, _: &mut dyn Vm| match exit {
4164 VcpuExit::Hlt => {
4165 hlt_count += 1;
4166 hlt_count > 1 // Halt execution after the second HLT
4167 }
4168 r => panic!("unexpected exit reason: {:?}", r),
4169 }
4170 );
4171 }
4172
4173 #[test]
test_minimal_exception_injection()4174 fn test_minimal_exception_injection() {
4175 // This test tries to write an invalid MSR, causing a General Protection exception to be
4176 // injected by the hypervisor (since MSR writes cause a VMEXIT). We run it in long mode since
4177 // real mode exception handling isn't always well supported (failed on Intel HAXM).
4178 mod assembly {
4179 use super::*;
4180
4181 // An ISR that handles any generic interrupt.
4182 global_asm_data!(
4183 pub isr_generic,
4184 ".code64",
4185 // Set EBX to 888 to observe this is where we halted.
4186 "mov ebx, 888",
4187 "hlt"
4188 );
4189
4190 // An ISR that handles the General Protection fault specifically.
4191 global_asm_data!(
4192 pub isr_gp,
4193 ".code64",
4194 // Set EBX to 999 to observe this is where we halted.
4195 "mov ebx, 999",
4196 "hlt"
4197 );
4198
4199 // Our VM entry (in long mode).
4200 global_asm_data!(
4201 pub init,
4202 ".code64",
4203 // Set up the stack, which will be used when CPU transfers the control to the ISR. If
4204 // not set up, can cause faults (stack should be aligned).
4205 "mov esp, 0x900",
4206 // We will verify EBX, set it here first.
4207 "mov ebx, 777",
4208 // Should trigger GP fault when we try to write to MSR 0.
4209 "wrmsr",
4210 // We should never get here since we halt in the fault handlers.
4211 "hlt",
4212 );
4213 }
4214
4215 let mem_size: u64 = 0x20000;
4216
4217 let setup = TestSetup {
4218 initial_regs: Regs {
4219 // WRMSR will try to write to ECX, we set it to zero to point to an old read-only MSR
4220 // (IA32_P5_MC_ADDR).
4221 rcx: 0,
4222 // Intentionally not setting IF flag since exceptions don't check it.
4223 rflags: 2,
4224 ..Default::default()
4225 },
4226 mem_size,
4227 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
4228 let start_addr: u64 = 0x1000;
4229 let guest_mem = vm.get_memory();
4230
4231 let isr_assembly = assembly::isr_generic::data().to_vec();
4232 let isr_assembly_len =
4233 u64::try_from(isr_assembly.len()).expect("ISR size should be within u64");
4234
4235 let isr_gp_assembly = assembly::isr_gp::data().to_vec();
4236 let isr_gp_assembly_len =
4237 u64::try_from(isr_gp_assembly.len()).expect("GP ISR size should be within u64");
4238
4239 let mut cur_addr = start_addr;
4240
4241 guest_mem
4242 .write_at_addr(&isr_assembly, GuestAddress(cur_addr))
4243 .expect("Failed to write ISR to guest memory");
4244 cur_addr += isr_assembly_len;
4245
4246 guest_mem
4247 .write_at_addr(&isr_gp_assembly, GuestAddress(cur_addr))
4248 .expect("Failed to write ISR to guest memory");
4249 cur_addr += isr_gp_assembly_len;
4250
4251 let mut regs = vcpu.get_regs().expect("Failed to get regs");
4252 regs.rip = cur_addr;
4253 vcpu.set_regs(®s).expect("Failed to set regs");
4254
4255 let init_assembly = assembly::init::data().to_vec();
4256 guest_mem
4257 .write_at_addr(&init_assembly, GuestAddress(cur_addr))
4258 .expect("Failed to write init assembly to guest memory");
4259
4260 let idt_entry_generic = IdtEntry64::new(start_addr);
4261 let idt_entry_gp = IdtEntry64::new(start_addr + isr_assembly_len);
4262
4263 let mut long_mode_config = ModeConfig::default_long_mode();
4264 long_mode_config
4265 .set_idt_long_mode((0..256).map(|i| {
4266 // GP handler is vector 13.
4267 if i == 0x0D {
4268 idt_entry_gp
4269 } else {
4270 idt_entry_generic
4271 }
4272 }))
4273 .set_idt_base_addr(0x12_000);
4274 long_mode_config.enter_long_mode(vcpu, vm);
4275 })),
4276 ..Default::default()
4277 };
4278
4279 run_tests!(
4280 setup,
4281 |_, regs, _| {
4282 // If EBX is 999 the GP handler ran.
4283 assert_eq!(regs.rbx, 999);
4284 },
4285 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
4286 );
4287 }
4288
4289 #[test]
test_pmode_segment_limit()4290 fn test_pmode_segment_limit() {
4291 // This test configures 32-bit protected mode and verifies that segment limits are converted
4292 // correctly. The test setup configures a segment with the 20-bit limit field set to 0xFFFFF and
4293 // the 4096-byte granularity bit set, which should result in a 4 GB limit (0xFFFFFFFF).
4294 mod assembly {
4295 use super::*;
4296
4297 global_asm_data!(
4298 pub init,
4299 ".code32",
4300 // Load the CS segment limit into EAX.
4301 "mov cx, cs",
4302 "lsl eax, cx",
4303 "hlt",
4304 );
4305 }
4306
4307 let mem_size: u64 = 0x20000;
4308
4309 let setup = TestSetup {
4310 initial_regs: Regs {
4311 ..Default::default()
4312 },
4313 mem_size,
4314 extra_vm_setup: Some(Box::new(|vcpu: &mut dyn VcpuX86_64, vm: &mut dyn Vm| {
4315 ModeConfig::default_protected_mode().enter_protected_mode(vcpu, vm);
4316
4317 let guest_mem = vm.get_memory();
4318
4319 let mut regs = vcpu.get_regs().expect("Failed to get regs");
4320 regs.rax = 12345;
4321 regs.rip = 0x1000;
4322 vcpu.set_regs(®s).expect("Failed to set regs");
4323
4324 let init_assembly = assembly::init::data().to_vec();
4325 guest_mem
4326 .write_at_addr(&init_assembly, GuestAddress(0x1000))
4327 .expect("Failed to write init assembly to guest memory");
4328 })),
4329 ..Default::default()
4330 };
4331
4332 run_tests!(
4333 setup,
4334 |_, regs, _| {
4335 // The output of the LSL instruction should be 4GB - 1.
4336 assert_eq!(regs.rax, 0xFFFFFFFF);
4337 },
4338 |_, exit, _, _: &mut dyn Vm| matches!(exit, VcpuExit::Hlt)
4339 );
4340 }
4341