Lines Matching +full:guest +full:- +full:index +full:- +full:bits
18 #include <linux/amd-iommu.h>
24 #include <linux/psp-sev.h>
40 #include <asm/spec-ctrl.h>
57 MODULE_DESCRIPTION("KVM support for SVM (AMD-V) extensions");
77 * are published and we know what the new status bits are
86 u32 index; /* Index of the MSR */ member
89 { .index = MSR_STAR, .always = true },
90 { .index = MSR_IA32_SYSENTER_CS, .always = true },
91 { .index = MSR_IA32_SYSENTER_EIP, .always = false },
92 { .index = MSR_IA32_SYSENTER_ESP, .always = false },
94 { .index = MSR_GS_BASE, .always = true },
95 { .index = MSR_FS_BASE, .always = true },
96 { .index = MSR_KERNEL_GS_BASE, .always = true },
97 { .index = MSR_LSTAR, .always = true },
98 { .index = MSR_CSTAR, .always = true },
99 { .index = MSR_SYSCALL_MASK, .always = true },
101 { .index = MSR_IA32_SPEC_CTRL, .always = false },
102 { .index = MSR_IA32_PRED_CMD, .always = false },
103 { .index = MSR_IA32_FLUSH_CMD, .always = false },
104 { .index = MSR_IA32_DEBUGCTLMSR, .always = false },
105 { .index = MSR_IA32_LASTBRANCHFROMIP, .always = false },
106 { .index = MSR_IA32_LASTBRANCHTOIP, .always = false },
107 { .index = MSR_IA32_LASTINTFROMIP, .always = false },
108 { .index = MSR_IA32_LASTINTTOIP, .always = false },
109 { .index = MSR_IA32_XSS, .always = false },
110 { .index = MSR_EFER, .always = false },
111 { .index = MSR_IA32_CR_PAT, .always = false },
112 { .index = MSR_AMD64_SEV_ES_GHCB, .always = true },
113 { .index = MSR_TSC_AUX, .always = false },
114 { .index = X2APIC_MSR(APIC_ID), .always = false },
115 { .index = X2APIC_MSR(APIC_LVR), .always = false },
116 { .index = X2APIC_MSR(APIC_TASKPRI), .always = false },
117 { .index = X2APIC_MSR(APIC_ARBPRI), .always = false },
118 { .index = X2APIC_MSR(APIC_PROCPRI), .always = false },
119 { .index = X2APIC_MSR(APIC_EOI), .always = false },
120 { .index = X2APIC_MSR(APIC_RRR), .always = false },
121 { .index = X2APIC_MSR(APIC_LDR), .always = false },
122 { .index = X2APIC_MSR(APIC_DFR), .always = false },
123 { .index = X2APIC_MSR(APIC_SPIV), .always = false },
124 { .index = X2APIC_MSR(APIC_ISR), .always = false },
125 { .index = X2APIC_MSR(APIC_TMR), .always = false },
126 { .index = X2APIC_MSR(APIC_IRR), .always = false },
127 { .index = X2APIC_MSR(APIC_ESR), .always = false },
128 { .index = X2APIC_MSR(APIC_ICR), .always = false },
129 { .index = X2APIC_MSR(APIC_ICR2), .always = false },
133 * AMD does not virtualize APIC TSC-deadline timer mode, but it is
138 { .index = X2APIC_MSR(APIC_LVTTHMR), .always = false },
139 { .index = X2APIC_MSR(APIC_LVTPC), .always = false },
140 { .index = X2APIC_MSR(APIC_LVT0), .always = false },
141 { .index = X2APIC_MSR(APIC_LVT1), .always = false },
142 { .index = X2APIC_MSR(APIC_LVTERR), .always = false },
143 { .index = X2APIC_MSR(APIC_TMICT), .always = false },
144 { .index = X2APIC_MSR(APIC_TMCCT), .always = false },
145 { .index = X2APIC_MSR(APIC_TDCR), .always = false },
146 { .index = MSR_INVALID, .always = false },
150 * These 2 parameters are used to config the controls for Pause-Loop Exiting:
162 * the amount of time a guest is allowed to execute in a pause loop.
163 * In this mode, a 16-bit pause filter threshold field is added in the
185 /* Default doubles per-vcpu window every exit. */
189 /* Default resets per-vcpu window every exit to pause_filter_count. */
259 static int tsc_aux_uret_slot __read_mostly = -1;
277 offset = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */ in svm_msrpm_offset()
280 /* Now we have the u8 offset - but need the u32 offset */ in svm_msrpm_offset()
300 u64 old_efer = vcpu->arch.efer; in svm_set_efer()
301 vcpu->arch.efer = efer; in svm_set_efer()
320 * Free the nested guest state, unless we are in SMM. in svm_set_efer()
321 * In this case we will return to the nested guest in svm_set_efer()
331 vcpu->arch.efer = old_efer; in svm_set_efer()
337 * decrypt guest memory to workaround the erratum. in svm_set_efer()
339 if (svm_gp_erratum_intercept && !sev_guest(vcpu->kvm)) in svm_set_efer()
344 svm->vmcb->save.efer = efer | EFER_SVME; in svm_set_efer()
345 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_set_efer()
354 if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) in svm_get_interrupt_shadow()
364 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK; in svm_set_interrupt_shadow()
366 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK; in svm_set_interrupt_shadow()
377 * SEV-ES does not expose the next RIP. The RIP update is controlled by in __svm_skip_emulated_instruction()
378 * the type of exit and the #VC handler in the guest. in __svm_skip_emulated_instruction()
380 if (sev_es_guest(vcpu->kvm)) in __svm_skip_emulated_instruction()
383 if (nrips && svm->vmcb->control.next_rip != 0) { in __svm_skip_emulated_instruction()
385 svm->next_rip = svm->vmcb->control.next_rip; in __svm_skip_emulated_instruction()
388 if (!svm->next_rip) { in __svm_skip_emulated_instruction()
390 old_rflags = svm->vmcb->save.rflags; in __svm_skip_emulated_instruction()
396 svm->vmcb->save.rflags = old_rflags; in __svm_skip_emulated_instruction()
398 kvm_rip_write(vcpu, svm->next_rip); in __svm_skip_emulated_instruction()
421 * the CPU was vectoring an INTO/INT3 in the guest. Temporarily skip in svm_update_soft_interrupt_rip()
424 * hardware will fail to advance guest RIP during event injection. in svm_update_soft_interrupt_rip()
431 return -EIO; in svm_update_soft_interrupt_rip()
437 * VMCB's next_rip will be lost (cleared on VM-Exit) if the injection in svm_update_soft_interrupt_rip()
438 * doesn't complete due to a VM-Exit occurring while the CPU is in svm_update_soft_interrupt_rip()
441 * being injected by L1 for L2, or if the guest is patching INT3 into in svm_update_soft_interrupt_rip()
444 svm->soft_int_injected = true; in svm_update_soft_interrupt_rip()
445 svm->soft_int_csbase = svm->vmcb->save.cs.base; in svm_update_soft_interrupt_rip()
446 svm->soft_int_old_rip = old_rip; in svm_update_soft_interrupt_rip()
447 svm->soft_int_next_rip = rip; in svm_update_soft_interrupt_rip()
453 svm->vmcb->control.next_rip = rip; in svm_update_soft_interrupt_rip()
460 struct kvm_queued_exception *ex = &vcpu->arch.exception; in svm_inject_exception()
465 if (kvm_exception_is_soft(ex->vector) && in svm_inject_exception()
469 svm->vmcb->control.event_inj = ex->vector in svm_inject_exception()
471 | (ex->has_error_code ? SVM_EVTINJ_VALID_ERR : 0) in svm_inject_exception()
473 svm->vmcb->control.event_inj_err = ex->error_code; in svm_inject_exception()
506 vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3; in svm_init_osvw()
507 vcpu->arch.osvw.status = osvw_status & ~(6ULL); in svm_init_osvw()
510 * By increasing VCPU's osvw.length to 3 we are telling the guest that in svm_init_osvw()
511 * all osvw.status bits inside that length, including bit 0 (which is in svm_init_osvw()
514 * be conservative here and therefore we tell the guest that erratum 298 in svm_init_osvw()
518 vcpu->arch.osvw.status |= 1; in svm_init_osvw()
526 if (c->x86_vendor != X86_VENDOR_AMD && in __kvm_is_svm_supported()
527 c->x86_vendor != X86_VENDOR_HYGON) { in __kvm_is_svm_supported()
538 pr_info("KVM is unsupported when running as an SEV guest\n"); in __kvm_is_svm_supported()
559 return -EIO; in svm_check_processor_compat()
575 return &sd->save_area->host_sev_es_save; in sev_es_host_save_area()
621 return -EBUSY; in svm_enable_virtualization_cpu()
624 sd->asid_generation = 1; in svm_enable_virtualization_cpu()
625 sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1; in svm_enable_virtualization_cpu()
626 sd->next_asid = sd->max_asid + 1; in svm_enable_virtualization_cpu()
627 sd->min_asid = max_sev_asid + 1; in svm_enable_virtualization_cpu()
631 wrmsrl(MSR_VM_HSAVE_PA, sd->save_area_pa); in svm_enable_virtualization_cpu()
643 * Get OSVW bits. in svm_enable_virtualization_cpu()
646 * revisions and therefore different OSVW bits. If bits are not the same in svm_enable_virtualization_cpu()
666 osvw_status &= (1ULL << osvw_len) - 1; in svm_enable_virtualization_cpu()
677 * "B" field (see sev_es_prepare_switch_to_guest()) for SEV-ES guests. in svm_enable_virtualization_cpu()
684 rdmsr(MSR_TSC_AUX, sev_es_host_save_area(sd)->tsc_aux, msr_hi); in svm_enable_virtualization_cpu()
694 if (!sd->save_area) in svm_cpu_uninit()
697 kfree(sd->sev_vmcbs); in svm_cpu_uninit()
698 __free_page(__sme_pa_to_page(sd->save_area_pa)); in svm_cpu_uninit()
699 sd->save_area_pa = 0; in svm_cpu_uninit()
700 sd->save_area = NULL; in svm_cpu_uninit()
707 int ret = -ENOMEM; in svm_cpu_init()
718 sd->save_area = page_address(save_area_page); in svm_cpu_init()
719 sd->save_area_pa = __sme_page_pa(save_area_page); in svm_cpu_init()
730 struct vmcb *vmcb = svm->vmcb01.ptr; in set_dr_intercepts()
732 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_READ); in set_dr_intercepts()
733 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_READ); in set_dr_intercepts()
734 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_READ); in set_dr_intercepts()
735 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_READ); in set_dr_intercepts()
736 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_READ); in set_dr_intercepts()
737 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_READ); in set_dr_intercepts()
738 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_READ); in set_dr_intercepts()
739 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR0_WRITE); in set_dr_intercepts()
740 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR1_WRITE); in set_dr_intercepts()
741 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR2_WRITE); in set_dr_intercepts()
742 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR3_WRITE); in set_dr_intercepts()
743 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR4_WRITE); in set_dr_intercepts()
744 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR5_WRITE); in set_dr_intercepts()
745 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR6_WRITE); in set_dr_intercepts()
746 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_READ); in set_dr_intercepts()
747 vmcb_set_intercept(&vmcb->control, INTERCEPT_DR7_WRITE); in set_dr_intercepts()
754 struct vmcb *vmcb = svm->vmcb01.ptr; in clr_dr_intercepts()
756 vmcb->control.intercepts[INTERCEPT_DR] = 0; in clr_dr_intercepts()
765 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) in direct_access_msr_slot()
766 if (direct_access_msrs[i].index == msr) in direct_access_msr_slot()
769 return -ENOENT; in direct_access_msr_slot()
778 if (slot == -ENOENT) in set_shadow_msr_intercept()
783 set_bit(slot, svm->shadow_msr_intercept.read); in set_shadow_msr_intercept()
785 clear_bit(slot, svm->shadow_msr_intercept.read); in set_shadow_msr_intercept()
788 set_bit(slot, svm->shadow_msr_intercept.write); in set_shadow_msr_intercept()
790 clear_bit(slot, svm->shadow_msr_intercept.write); in set_shadow_msr_intercept()
793 static bool valid_msr_intercept(u32 index) in valid_msr_intercept() argument
795 return direct_access_msr_slot(index) != -ENOENT; in valid_msr_intercept()
806 * For non-nested case: in msr_write_intercepted()
814 msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm: in msr_write_intercepted()
815 to_svm(vcpu)->msrpm; in msr_write_intercepted()
860 svm->nested.force_msr_bitmap_recalc = true; in set_msr_interception_bitmap()
889 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { in svm_vcpu_init_msrpm()
892 set_msr_interception(vcpu, msrpm, direct_access_msrs[i].index, 1, 1); in svm_vcpu_init_msrpm()
900 if (intercept == svm->x2avic_msrs_intercepted) in svm_set_x2apic_msr_interception()
907 int index = direct_access_msrs[i].index; in svm_set_x2apic_msr_interception() local
909 if ((index < APIC_BASE_MSR) || in svm_set_x2apic_msr_interception()
910 (index > APIC_BASE_MSR + 0xff)) in svm_set_x2apic_msr_interception()
912 set_msr_interception(&svm->vcpu, svm->msrpm, index, in svm_set_x2apic_msr_interception()
916 svm->x2avic_msrs_intercepted = intercept; in svm_set_x2apic_msr_interception()
934 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { in svm_msr_filter_changed()
935 u32 msr = direct_access_msrs[i].index; in svm_msr_filter_changed()
936 u32 read = test_bit(i, svm->shadow_msr_intercept.read); in svm_msr_filter_changed()
937 u32 write = test_bit(i, svm->shadow_msr_intercept.write); in svm_msr_filter_changed()
939 set_msr_interception_bitmap(vcpu, svm->msrpm, msr, read, write); in svm_msr_filter_changed()
976 for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) { in init_msrpm_offsets()
979 offset = svm_msrpm_offset(direct_access_msrs[i].index); in init_msrpm_offsets()
988 to_vmcb->save.dbgctl = from_vmcb->save.dbgctl; in svm_copy_lbrs()
989 to_vmcb->save.br_from = from_vmcb->save.br_from; in svm_copy_lbrs()
990 to_vmcb->save.br_to = from_vmcb->save.br_to; in svm_copy_lbrs()
991 to_vmcb->save.last_excp_from = from_vmcb->save.last_excp_from; in svm_copy_lbrs()
992 to_vmcb->save.last_excp_to = from_vmcb->save.last_excp_to; in svm_copy_lbrs()
1001 svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK; in svm_enable_lbrv()
1002 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1); in svm_enable_lbrv()
1003 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1); in svm_enable_lbrv()
1004 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 1, 1); in svm_enable_lbrv()
1005 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 1, 1); in svm_enable_lbrv()
1007 if (sev_es_guest(vcpu->kvm)) in svm_enable_lbrv()
1008 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_DEBUGCTLMSR, 1, 1); in svm_enable_lbrv()
1010 /* Move the LBR msrs to the vmcb02 so that the guest can see them. */ in svm_enable_lbrv()
1012 svm_copy_lbrs(svm->vmcb, svm->vmcb01.ptr); in svm_enable_lbrv()
1019 KVM_BUG_ON(sev_es_guest(vcpu->kvm), vcpu->kvm); in svm_disable_lbrv()
1021 svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK; in svm_disable_lbrv()
1022 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0); in svm_disable_lbrv()
1023 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0); in svm_disable_lbrv()
1024 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTFROMIP, 0, 0); in svm_disable_lbrv()
1025 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_LASTINTTOIP, 0, 0); in svm_disable_lbrv()
1029 * on nested guest entries. in svm_disable_lbrv()
1032 svm_copy_lbrs(svm->vmcb01.ptr, svm->vmcb); in svm_disable_lbrv()
1042 return svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK ? svm->vmcb : in svm_get_lbr_vmcb()
1043 svm->vmcb01.ptr; in svm_get_lbr_vmcb()
1049 bool current_enable_lbrv = svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK; in svm_update_lbrv()
1050 bool enable_lbrv = (svm_get_lbr_vmcb(svm)->save.dbgctl & DEBUGCTLMSR_LBR) || in svm_update_lbrv()
1052 (svm->nested.ctl.virt_ext & LBR_CTL_ENABLE_MASK)); in svm_update_lbrv()
1065 svm->nmi_singlestep = false; in disable_nmi_singlestep()
1067 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) { in disable_nmi_singlestep()
1068 /* Clear our flags if they were not set by the guest */ in disable_nmi_singlestep()
1069 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) in disable_nmi_singlestep()
1070 svm->vmcb->save.rflags &= ~X86_EFLAGS_TF; in disable_nmi_singlestep()
1071 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) in disable_nmi_singlestep()
1072 svm->vmcb->save.rflags &= ~X86_EFLAGS_RF; in disable_nmi_singlestep()
1079 struct vmcb_control_area *control = &svm->vmcb->control; in grow_ple_window()
1080 int old = control->pause_filter_count; in grow_ple_window()
1082 if (kvm_pause_in_guest(vcpu->kvm)) in grow_ple_window()
1085 control->pause_filter_count = __grow_ple_window(old, in grow_ple_window()
1090 if (control->pause_filter_count != old) { in grow_ple_window()
1091 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in grow_ple_window()
1092 trace_kvm_ple_window_update(vcpu->vcpu_id, in grow_ple_window()
1093 control->pause_filter_count, old); in grow_ple_window()
1100 struct vmcb_control_area *control = &svm->vmcb->control; in shrink_ple_window()
1101 int old = control->pause_filter_count; in shrink_ple_window()
1103 if (kvm_pause_in_guest(vcpu->kvm)) in shrink_ple_window()
1106 control->pause_filter_count = in shrink_ple_window()
1111 if (control->pause_filter_count != old) { in shrink_ple_window()
1112 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in shrink_ple_window()
1113 trace_kvm_ple_window_update(vcpu->vcpu_id, in shrink_ple_window()
1114 control->pause_filter_count, old); in shrink_ple_window()
1133 seg->selector = 0; in init_seg()
1134 seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK | in init_seg()
1136 seg->limit = 0xffff; in init_seg()
1137 seg->base = 0; in init_seg()
1142 seg->selector = 0; in init_sys_seg()
1143 seg->attrib = SVM_SELECTOR_P_MASK | type; in init_sys_seg()
1144 seg->limit = 0xffff; in init_sys_seg()
1145 seg->base = 0; in init_sys_seg()
1152 return svm->nested.ctl.tsc_offset; in svm_get_l2_tsc_offset()
1159 return svm->tsc_ratio_msr; in svm_get_l2_tsc_multiplier()
1166 svm->vmcb01.ptr->control.tsc_offset = vcpu->arch.l1_tsc_offset; in svm_write_tsc_offset()
1167 svm->vmcb->control.tsc_offset = vcpu->arch.tsc_offset; in svm_write_tsc_offset()
1168 vmcb_mark_dirty(svm->vmcb, VMCB_INTERCEPTS); in svm_write_tsc_offset()
1174 if (to_svm(vcpu)->guest_state_loaded) in svm_write_tsc_multiplier()
1175 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); in svm_write_tsc_multiplier()
1179 /* Evaluate instruction intercepts that depend on guest CPUID features. */
1185 * roots, or if INVPCID is disabled in the guest to inject #UD. in svm_recalc_instruction_intercepts()
1189 !guest_cpu_cap_has(&svm->vcpu, X86_FEATURE_INVPCID)) in svm_recalc_instruction_intercepts()
1210 * accesses because the processor only stores 32 bits. in init_vmcb_after_set_cpuid()
1215 svm->vmcb->control.virt_ext &= ~VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; in init_vmcb_after_set_cpuid()
1217 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 0, 0); in init_vmcb_after_set_cpuid()
1218 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 0, 0); in init_vmcb_after_set_cpuid()
1227 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK; in init_vmcb_after_set_cpuid()
1230 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_EIP, 1, 1); in init_vmcb_after_set_cpuid()
1231 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SYSENTER_ESP, 1, 1); in init_vmcb_after_set_cpuid()
1238 struct vmcb *vmcb = svm->vmcb01.ptr; in init_vmcb()
1239 struct vmcb_control_area *control = &vmcb->control; in init_vmcb()
1240 struct vmcb_save_area *save = &vmcb->save; in init_vmcb()
1259 * Guest access to VMware backdoor ports could legitimately in init_vmcb()
1295 if (!kvm_mwait_in_guest(vcpu->kvm)) { in init_vmcb()
1300 if (!kvm_hlt_in_guest(vcpu->kvm)) in init_vmcb()
1303 control->iopm_base_pa = iopm_base; in init_vmcb()
1304 control->msrpm_base_pa = __sme_set(__pa(svm->msrpm)); in init_vmcb()
1305 control->int_ctl = V_INTR_MASKING_MASK; in init_vmcb()
1307 init_seg(&save->es); in init_vmcb()
1308 init_seg(&save->ss); in init_vmcb()
1309 init_seg(&save->ds); in init_vmcb()
1310 init_seg(&save->fs); in init_vmcb()
1311 init_seg(&save->gs); in init_vmcb()
1313 save->cs.selector = 0xf000; in init_vmcb()
1314 save->cs.base = 0xffff0000; in init_vmcb()
1316 save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK | in init_vmcb()
1318 save->cs.limit = 0xffff; in init_vmcb()
1320 save->gdtr.base = 0; in init_vmcb()
1321 save->gdtr.limit = 0xffff; in init_vmcb()
1322 save->idtr.base = 0; in init_vmcb()
1323 save->idtr.limit = 0xffff; in init_vmcb()
1325 init_sys_seg(&save->ldtr, SEG_TYPE_LDT); in init_vmcb()
1326 init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16); in init_vmcb()
1330 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE; in init_vmcb()
1335 save->g_pat = vcpu->arch.pat; in init_vmcb()
1336 save->cr3 = 0; in init_vmcb()
1338 svm->current_vmcb->asid_generation = 0; in init_vmcb()
1339 svm->asid = 0; in init_vmcb()
1341 svm->nested.vmcb12_gpa = INVALID_GPA; in init_vmcb()
1342 svm->nested.last_vmcb12_gpa = INVALID_GPA; in init_vmcb()
1344 if (!kvm_pause_in_guest(vcpu->kvm)) { in init_vmcb()
1345 control->pause_filter_count = pause_filter_count; in init_vmcb()
1347 control->pause_filter_thresh = pause_filter_thresh; in init_vmcb()
1360 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); in init_vmcb()
1366 svm->vmcb->control.int_ctl |= V_NMI_ENABLE_MASK; in init_vmcb()
1371 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK; in init_vmcb()
1374 if (sev_guest(vcpu->kvm)) in init_vmcb()
1389 svm_vcpu_init_msrpm(vcpu, svm->msrpm); in __svm_vcpu_reset()
1393 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_STUFF_FEATURE_MSRS)) in __svm_vcpu_reset()
1394 vcpu->arch.microcode_version = 0x01000065; in __svm_vcpu_reset()
1395 svm->tsc_ratio_msr = kvm_caps.default_tsc_scaling_ratio; in __svm_vcpu_reset()
1397 svm->nmi_masked = false; in __svm_vcpu_reset()
1398 svm->awaiting_iret_completion = false; in __svm_vcpu_reset()
1400 if (sev_es_guest(vcpu->kvm)) in __svm_vcpu_reset()
1408 svm->spec_ctrl = 0; in svm_vcpu_reset()
1409 svm->virt_spec_ctrl = 0; in svm_vcpu_reset()
1422 svm->current_vmcb = target_vmcb; in svm_switch_vmcb()
1423 svm->vmcb = target_vmcb->ptr; in svm_switch_vmcb()
1436 err = -ENOMEM; in svm_vcpu_create()
1441 if (sev_es_guest(vcpu->kvm)) { in svm_vcpu_create()
1443 * SEV-ES guests require a separate VMSA page used to contain in svm_vcpu_create()
1444 * the encrypted register state of the guest. in svm_vcpu_create()
1455 svm->msrpm = svm_vcpu_alloc_msrpm(); in svm_vcpu_create()
1456 if (!svm->msrpm) { in svm_vcpu_create()
1457 err = -ENOMEM; in svm_vcpu_create()
1461 svm->x2avic_msrs_intercepted = true; in svm_vcpu_create()
1463 svm->vmcb01.ptr = page_address(vmcb01_page); in svm_vcpu_create()
1464 svm->vmcb01.pa = __sme_set(page_to_pfn(vmcb01_page) << PAGE_SHIFT); in svm_vcpu_create()
1465 svm_switch_vmcb(svm, &svm->vmcb01); in svm_vcpu_create()
1468 svm->sev_es.vmsa = page_address(vmsa_page); in svm_vcpu_create()
1470 svm->guest_state_loaded = false; in svm_vcpu_create()
1500 svm_clear_current_vmcb(svm->vmcb); in svm_vcpu_free()
1507 __free_page(__sme_pa_to_page(svm->vmcb01.pa)); in svm_vcpu_free()
1508 __free_pages(virt_to_page(svm->msrpm), get_order(MSRPM_SIZE)); in svm_vcpu_free()
1514 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in svm_prepare_switch_to_guest()
1516 if (sev_es_guest(vcpu->kvm)) in svm_prepare_switch_to_guest()
1519 if (svm->guest_state_loaded) in svm_prepare_switch_to_guest()
1523 * Save additional host state that will be restored on VMEXIT (sev-es) in svm_prepare_switch_to_guest()
1526 vmsave(sd->save_area_pa); in svm_prepare_switch_to_guest()
1527 if (sev_es_guest(vcpu->kvm)) in svm_prepare_switch_to_guest()
1531 __svm_write_tsc_multiplier(vcpu->arch.tsc_scaling_ratio); in svm_prepare_switch_to_guest()
1534 * TSC_AUX is always virtualized for SEV-ES guests when the feature is in svm_prepare_switch_to_guest()
1540 (!boot_cpu_has(X86_FEATURE_V_TSC_AUX) || !sev_es_guest(vcpu->kvm))) in svm_prepare_switch_to_guest()
1541 kvm_set_user_return_msr(tsc_aux_uret_slot, svm->tsc_aux, -1ull); in svm_prepare_switch_to_guest()
1543 svm->guest_state_loaded = true; in svm_prepare_switch_to_guest()
1548 to_svm(vcpu)->guest_state_loaded = false; in svm_prepare_host_switch()
1556 if (vcpu->scheduled_out && !kvm_pause_in_guest(vcpu->kvm)) in svm_vcpu_load()
1559 if (sd->current_vmcb != svm->vmcb) { in svm_vcpu_load()
1560 sd->current_vmcb = svm->vmcb; in svm_vcpu_load()
1576 ++vcpu->stat.host_state_reload; in svm_vcpu_put()
1582 unsigned long rflags = svm->vmcb->save.rflags; in svm_get_rflags()
1584 if (svm->nmi_singlestep) { in svm_get_rflags()
1585 /* Hide our flags if they were not set by the guest */ in svm_get_rflags()
1586 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF)) in svm_get_rflags()
1588 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF)) in svm_get_rflags()
1596 if (to_svm(vcpu)->nmi_singlestep) in svm_set_rflags()
1601 * (caused by either a task switch or an inter-privilege IRET), in svm_set_rflags()
1604 to_svm(vcpu)->vmcb->save.rflags = rflags; in svm_set_rflags()
1609 struct vmcb *vmcb = to_svm(vcpu)->vmcb; in svm_get_if_flag()
1611 return sev_es_guest(vcpu->kvm) in svm_get_if_flag()
1612 ? vmcb->control.int_state & SVM_GUEST_INTERRUPT_MASK in svm_get_if_flag()
1623 * When !npt_enabled, mmu->pdptrs[] is already available since in svm_cache_reg()
1630 KVM_BUG_ON(1, vcpu->kvm); in svm_cache_reg()
1641 WARN_ON(kvm_vcpu_apicv_activated(&svm->vcpu)); in svm_set_vintr()
1659 control = &svm->vmcb->control; in svm_set_vintr()
1660 control->int_vector = 0x0; in svm_set_vintr()
1661 control->int_ctl &= ~V_INTR_PRIO_MASK; in svm_set_vintr()
1662 control->int_ctl |= V_IRQ_MASK | in svm_set_vintr()
1663 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT); in svm_set_vintr()
1664 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_set_vintr()
1672 svm->vmcb->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; in svm_clear_vintr()
1673 if (is_guest_mode(&svm->vcpu)) { in svm_clear_vintr()
1674 svm->vmcb01.ptr->control.int_ctl &= ~V_IRQ_INJECTION_BITS_MASK; in svm_clear_vintr()
1676 WARN_ON((svm->vmcb->control.int_ctl & V_TPR_MASK) != in svm_clear_vintr()
1677 (svm->nested.ctl.int_ctl & V_TPR_MASK)); in svm_clear_vintr()
1679 svm->vmcb->control.int_ctl |= svm->nested.ctl.int_ctl & in svm_clear_vintr()
1682 svm->vmcb->control.int_vector = svm->nested.ctl.int_vector; in svm_clear_vintr()
1685 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_clear_vintr()
1690 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; in svm_seg()
1691 struct vmcb_save_area *save01 = &to_svm(vcpu)->vmcb01.ptr->save; in svm_seg()
1694 case VCPU_SREG_CS: return &save->cs; in svm_seg()
1695 case VCPU_SREG_DS: return &save->ds; in svm_seg()
1696 case VCPU_SREG_ES: return &save->es; in svm_seg()
1697 case VCPU_SREG_FS: return &save01->fs; in svm_seg()
1698 case VCPU_SREG_GS: return &save01->gs; in svm_seg()
1699 case VCPU_SREG_SS: return &save->ss; in svm_seg()
1700 case VCPU_SREG_TR: return &save01->tr; in svm_seg()
1701 case VCPU_SREG_LDTR: return &save01->ldtr; in svm_seg()
1711 return s->base; in svm_get_segment_base()
1719 var->base = s->base; in svm_get_segment()
1720 var->limit = s->limit; in svm_get_segment()
1721 var->selector = s->selector; in svm_get_segment()
1722 var->type = s->attrib & SVM_SELECTOR_TYPE_MASK; in svm_get_segment()
1723 var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1; in svm_get_segment()
1724 var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3; in svm_get_segment()
1725 var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1; in svm_get_segment()
1726 var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1; in svm_get_segment()
1727 var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1; in svm_get_segment()
1728 var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1; in svm_get_segment()
1735 * running KVM nested. It also helps cross-vendor migration, because in svm_get_segment()
1738 var->g = s->limit > 0xfffff; in svm_get_segment()
1744 var->unusable = !var->present; in svm_get_segment()
1752 var->type |= 0x2; in svm_get_segment()
1763 * cross-vendor migration. in svm_get_segment()
1765 if (!var->unusable) in svm_get_segment()
1766 var->type |= 0x1; in svm_get_segment()
1775 if (var->unusable) in svm_get_segment()
1776 var->db = 0; in svm_get_segment()
1778 var->dpl = to_svm(vcpu)->vmcb->save.cpl; in svm_get_segment()
1785 struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save; in svm_get_cpl()
1787 return save->cpl; in svm_get_cpl()
1803 dt->size = svm->vmcb->save.idtr.limit; in svm_get_idt()
1804 dt->address = svm->vmcb->save.idtr.base; in svm_get_idt()
1811 svm->vmcb->save.idtr.limit = dt->size; in svm_set_idt()
1812 svm->vmcb->save.idtr.base = dt->address ; in svm_set_idt()
1813 vmcb_mark_dirty(svm->vmcb, VMCB_DT); in svm_set_idt()
1820 dt->size = svm->vmcb->save.gdtr.limit; in svm_get_gdt()
1821 dt->address = svm->vmcb->save.gdtr.base; in svm_get_gdt()
1828 svm->vmcb->save.gdtr.limit = dt->size; in svm_set_gdt()
1829 svm->vmcb->save.gdtr.base = dt->address ; in svm_set_gdt()
1830 vmcb_mark_dirty(svm->vmcb, VMCB_DT); in svm_set_gdt()
1839 * handled via kvm_mmu_load() while entering the guest. For guests in sev_post_set_cr3()
1840 * that do (SEV-ES/SEV-SNP), the cr3 update needs to be written to in sev_post_set_cr3()
1845 if (sev_es_guest(vcpu->kvm)) { in sev_post_set_cr3()
1846 svm->vmcb->save.cr3 = cr3; in sev_post_set_cr3()
1847 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in sev_post_set_cr3()
1863 if (vcpu->arch.efer & EFER_LME) { in svm_set_cr0()
1865 vcpu->arch.efer |= EFER_LMA; in svm_set_cr0()
1866 if (!vcpu->arch.guest_state_protected) in svm_set_cr0()
1867 svm->vmcb->save.efer |= EFER_LMA | EFER_LME; in svm_set_cr0()
1871 vcpu->arch.efer &= ~EFER_LMA; in svm_set_cr0()
1872 if (!vcpu->arch.guest_state_protected) in svm_set_cr0()
1873 svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME); in svm_set_cr0()
1877 vcpu->arch.cr0 = cr0; in svm_set_cr0()
1886 * re-enable caching here because the QEMU bios in svm_set_cr0()
1887 * does not do it - this results in some delay at in svm_set_cr0()
1890 if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED)) in svm_set_cr0()
1893 svm->vmcb->save.cr0 = hcr0; in svm_set_cr0()
1894 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_set_cr0()
1897 * SEV-ES guests must always keep the CR intercepts cleared. CR in svm_set_cr0()
1900 if (sev_es_guest(vcpu->kvm)) in svm_set_cr0()
1921 unsigned long old_cr4 = vcpu->arch.cr4; in svm_set_cr4()
1923 vcpu->arch.cr4 = cr4; in svm_set_cr4()
1931 to_svm(vcpu)->vmcb->save.cr4 = cr4; in svm_set_cr4()
1932 vmcb_mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR); in svm_set_cr4()
1944 s->base = var->base; in svm_set_segment()
1945 s->limit = var->limit; in svm_set_segment()
1946 s->selector = var->selector; in svm_set_segment()
1947 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK); in svm_set_segment()
1948 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT; in svm_set_segment()
1949 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT; in svm_set_segment()
1950 s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT; in svm_set_segment()
1951 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT; in svm_set_segment()
1952 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT; in svm_set_segment()
1953 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT; in svm_set_segment()
1954 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT; in svm_set_segment()
1964 svm->vmcb->save.cpl = (var->dpl & 3); in svm_set_segment()
1966 vmcb_mark_dirty(svm->vmcb, VMCB_SEG); in svm_set_segment()
1975 if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) { in svm_update_exception_bitmap()
1976 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP) in svm_update_exception_bitmap()
1983 if (sd->next_asid > sd->max_asid) { in new_asid()
1984 ++sd->asid_generation; in new_asid()
1985 sd->next_asid = sd->min_asid; in new_asid()
1986 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID; in new_asid()
1987 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); in new_asid()
1990 svm->current_vmcb->asid_generation = sd->asid_generation; in new_asid()
1991 svm->asid = sd->next_asid++; in new_asid()
1996 struct vmcb *vmcb = to_svm(vcpu)->vmcb; in svm_set_dr6()
1998 if (vcpu->arch.guest_state_protected) in svm_set_dr6()
2001 if (unlikely(value != vmcb->save.dr6)) { in svm_set_dr6()
2002 vmcb->save.dr6 = value; in svm_set_dr6()
2011 if (WARN_ON_ONCE(sev_es_guest(vcpu->kvm))) in svm_sync_dirty_debug_regs()
2014 get_debugreg(vcpu->arch.db[0], 0); in svm_sync_dirty_debug_regs()
2015 get_debugreg(vcpu->arch.db[1], 1); in svm_sync_dirty_debug_regs()
2016 get_debugreg(vcpu->arch.db[2], 2); in svm_sync_dirty_debug_regs()
2017 get_debugreg(vcpu->arch.db[3], 3); in svm_sync_dirty_debug_regs()
2019 * We cannot reset svm->vmcb->save.dr6 to DR6_ACTIVE_LOW here, in svm_sync_dirty_debug_regs()
2022 vcpu->arch.dr6 = svm->vmcb->save.dr6; in svm_sync_dirty_debug_regs()
2023 vcpu->arch.dr7 = svm->vmcb->save.dr7; in svm_sync_dirty_debug_regs()
2024 vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT; in svm_sync_dirty_debug_regs()
2032 if (vcpu->arch.guest_state_protected) in svm_set_dr7()
2035 svm->vmcb->save.dr7 = value; in svm_set_dr7()
2036 vmcb_mark_dirty(svm->vmcb, VMCB_DR); in svm_set_dr7()
2043 u64 fault_address = svm->vmcb->control.exit_info_2; in pf_interception()
2044 u64 error_code = svm->vmcb->control.exit_info_1; in pf_interception()
2048 svm->vmcb->control.insn_bytes : NULL, in pf_interception()
2049 svm->vmcb->control.insn_len); in pf_interception()
2057 u64 fault_address = svm->vmcb->control.exit_info_2; in npf_interception()
2058 u64 error_code = svm->vmcb->control.exit_info_1; in npf_interception()
2062 * with KVM-defined sythentic flags. Clear the flags and continue on, in npf_interception()
2069 if (sev_snp_guest(vcpu->kvm) && (error_code & PFERR_GUEST_ENC_MASK)) in npf_interception()
2075 svm->vmcb->control.insn_bytes : NULL, in npf_interception()
2076 svm->vmcb->control.insn_len); in npf_interception()
2086 struct kvm_run *kvm_run = vcpu->run; in db_interception()
2089 if (!(vcpu->guest_debug & in db_interception()
2091 !svm->nmi_singlestep) { in db_interception()
2092 u32 payload = svm->vmcb->save.dr6 ^ DR6_ACTIVE_LOW; in db_interception()
2097 if (svm->nmi_singlestep) { in db_interception()
2103 if (vcpu->guest_debug & in db_interception()
2105 kvm_run->exit_reason = KVM_EXIT_DEBUG; in db_interception()
2106 kvm_run->debug.arch.dr6 = svm->vmcb->save.dr6; in db_interception()
2107 kvm_run->debug.arch.dr7 = svm->vmcb->save.dr7; in db_interception()
2108 kvm_run->debug.arch.pc = in db_interception()
2109 svm->vmcb->save.cs.base + svm->vmcb->save.rip; in db_interception()
2110 kvm_run->debug.arch.exception = DB_VECTOR; in db_interception()
2120 struct kvm_run *kvm_run = vcpu->run; in bp_interception()
2122 kvm_run->exit_reason = KVM_EXIT_DEBUG; in bp_interception()
2123 kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip; in bp_interception()
2124 kvm_run->debug.arch.exception = BP_VECTOR; in bp_interception()
2172 /* Flush tlb to evict multi-match entries */ in is_erratum_383()
2182 * Erratum 383 triggered. Guest state is corrupt so kill the in svm_handle_mce()
2183 * guest. in svm_handle_mce()
2185 pr_err("Guest triggered AMD Erratum 383\n"); in svm_handle_mce()
2206 struct kvm_run *kvm_run = vcpu->run; in shutdown_interception()
2218 * The VM save area for SEV-ES guests has already been encrypted so it in shutdown_interception()
2221 if (!sev_es_guest(vcpu->kvm)) { in shutdown_interception()
2222 clear_page(svm->vmcb); in shutdown_interception()
2226 kvm_run->exit_reason = KVM_EXIT_SHUTDOWN; in shutdown_interception()
2233 u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */ in io_interception()
2237 ++vcpu->stat.io_exits; in io_interception()
2244 if (sev_es_guest(vcpu->kvm)) in io_interception()
2250 svm->next_rip = svm->vmcb->control.exit_info_2; in io_interception()
2267 ++vcpu->stat.irq_exits; in intr_interception()
2281 ret = kvm_vcpu_map(vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map); in vmload_vmsave_interception()
2283 if (ret == -EINVAL) in vmload_vmsave_interception()
2293 svm_copy_vmloadsave_state(svm->vmcb, vmcb12); in vmload_vmsave_interception()
2294 svm->sysenter_eip_hi = 0; in vmload_vmsave_interception()
2295 svm->sysenter_esp_hi = 0; in vmload_vmsave_interception()
2297 svm_copy_vmloadsave_state(vmcb12, svm->vmcb); in vmload_vmsave_interception()
2333 struct x86_emulate_ctxt *ctxt = vcpu->arch.emulate_ctxt; in svm_instr_opcode()
2335 if (ctxt->b != 0x1 || ctxt->opcode_len != 2) in svm_instr_opcode()
2338 switch (ctxt->modrm) { in svm_instr_opcode()
2368 /* Returns '1' or -errno on failure, '0' on success. */ in emulate_svm_instr()
2380 * 1) SVM VM-related instructions (VMRUN/VMSAVE/VMLOAD) that trigger #GP on
2388 u32 error_code = svm->vmcb->control.exit_info_1; in gp_interception()
2414 if (svm->vmcb->save.rax & ~PAGE_MASK) in gp_interception()
2440 if (svm->vcpu.arch.smi_pending || in svm_set_gif()
2441 svm->vcpu.arch.nmi_pending || in svm_set_gif()
2442 kvm_cpu_has_injectable_intr(&svm->vcpu) || in svm_set_gif()
2443 kvm_apic_has_pending_init_or_sipi(&svm->vcpu)) in svm_set_gif()
2444 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu); in svm_set_gif()
2491 trace_kvm_invlpga(to_svm(vcpu)->vmcb->save.rip, asid, gva); in invlpga_interception()
2501 trace_kvm_skinit(to_svm(vcpu)->vmcb->save.rip, kvm_rax_read(vcpu)); in skinit_interception()
2512 int int_type = svm->vmcb->control.exit_int_info & in task_switch_interception()
2514 int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK; in task_switch_interception()
2516 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK; in task_switch_interception()
2518 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID; in task_switch_interception()
2522 tss_selector = (u16)svm->vmcb->control.exit_info_1; in task_switch_interception()
2524 if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2527 else if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2538 vcpu->arch.nmi_injected = false; in task_switch_interception()
2541 if (svm->vmcb->control.exit_info_2 & in task_switch_interception()
2545 (u32)svm->vmcb->control.exit_info_2; in task_switch_interception()
2567 int_vec = -1; in task_switch_interception()
2575 if (!sev_es_guest(svm->vcpu.kvm)) in svm_clr_iret_intercept()
2581 if (!sev_es_guest(svm->vcpu.kvm)) in svm_set_iret_intercept()
2589 WARN_ON_ONCE(sev_es_guest(vcpu->kvm)); in iret_interception()
2591 ++vcpu->stat.nmi_window_exits; in iret_interception()
2592 svm->awaiting_iret_completion = true; in iret_interception()
2595 svm->nmi_iret_rip = kvm_rip_read(vcpu); in iret_interception()
2606 kvm_mmu_invlpg(vcpu, to_svm(vcpu)->vmcb->control.exit_info_1); in invlpg_interception()
2624 unsigned long cr0 = vcpu->arch.cr0; in check_selective_cr0_intercepted()
2628 (!(vmcb12_is_intercept(&svm->nested.ctl, INTERCEPT_SELECTIVE_CR0)))) in check_selective_cr0_intercepted()
2635 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE; in check_selective_cr0_intercepted()
2654 if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0)) in cr_interception()
2657 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; in cr_interception()
2658 if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE) in cr_interception()
2659 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0; in cr_interception()
2661 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0; in cr_interception()
2665 cr -= 16; in cr_interception()
2696 val = vcpu->arch.cr2; in cr_interception()
2725 new_value = (unsigned long)svm->vmcb->control.exit_info_1; in cr_trap()
2727 cr = svm->vmcb->control.exit_code - SVM_EXIT_CR0_WRITE_TRAP; in cr_trap()
2760 * SEV-ES intercepts DR7 only to disable guest debugging and the guest issues a VMGEXIT in dr_interception()
2763 if (sev_es_guest(vcpu->kvm)) in dr_interception()
2766 if (vcpu->guest_debug == 0) { in dr_interception()
2773 vcpu->arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT; in dr_interception()
2780 reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK; in dr_interception()
2781 dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0; in dr_interception()
2783 dr -= 16; in dr_interception()
2803 vcpu->run->exit_reason = KVM_EXIT_SET_TPR; in cr8_write_interception()
2815 * whether the guest has X86_FEATURE_SVM - this avoids a failure if in efer_trap()
2816 * the guest doesn't have X86_FEATURE_SVM. in efer_trap()
2819 msr_info.index = MSR_EFER; in efer_trap()
2820 msr_info.data = to_svm(vcpu)->vmcb->control.exit_info_1 & ~EFER_SVME; in efer_trap()
2845 return sev_es_guest(vcpu->kvm) && in sev_es_prevent_msr_access()
2846 vcpu->arch.guest_state_protected && in sev_es_prevent_msr_access()
2847 svm_msrpm_offset(msr_info->index) != MSR_INVALID && in sev_es_prevent_msr_access()
2848 !msr_write_intercepted(vcpu, msr_info->index); in sev_es_prevent_msr_access()
2856 msr_info->data = 0; in svm_get_msr()
2857 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in svm_get_msr()
2860 switch (msr_info->index) { in svm_get_msr()
2862 if (!msr_info->host_initiated && in svm_get_msr()
2865 msr_info->data = svm->tsc_ratio_msr; in svm_get_msr()
2868 msr_info->data = svm->vmcb01.ptr->save.star; in svm_get_msr()
2872 msr_info->data = svm->vmcb01.ptr->save.lstar; in svm_get_msr()
2875 msr_info->data = svm->vmcb01.ptr->save.cstar; in svm_get_msr()
2878 msr_info->data = svm->vmcb01.ptr->save.gs.base; in svm_get_msr()
2881 msr_info->data = svm->vmcb01.ptr->save.fs.base; in svm_get_msr()
2884 msr_info->data = svm->vmcb01.ptr->save.kernel_gs_base; in svm_get_msr()
2887 msr_info->data = svm->vmcb01.ptr->save.sfmask; in svm_get_msr()
2891 msr_info->data = svm->vmcb01.ptr->save.sysenter_cs; in svm_get_msr()
2894 msr_info->data = (u32)svm->vmcb01.ptr->save.sysenter_eip; in svm_get_msr()
2896 msr_info->data |= (u64)svm->sysenter_eip_hi << 32; in svm_get_msr()
2899 msr_info->data = svm->vmcb01.ptr->save.sysenter_esp; in svm_get_msr()
2901 msr_info->data |= (u64)svm->sysenter_esp_hi << 32; in svm_get_msr()
2904 msr_info->data = svm->tsc_aux; in svm_get_msr()
2907 msr_info->data = svm_get_lbr_vmcb(svm)->save.dbgctl; in svm_get_msr()
2910 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_from; in svm_get_msr()
2913 msr_info->data = svm_get_lbr_vmcb(svm)->save.br_to; in svm_get_msr()
2916 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_from; in svm_get_msr()
2919 msr_info->data = svm_get_lbr_vmcb(svm)->save.last_excp_to; in svm_get_msr()
2922 msr_info->data = svm->nested.hsave_msr; in svm_get_msr()
2925 msr_info->data = svm->nested.vm_cr_msr; in svm_get_msr()
2928 if (!msr_info->host_initiated && in svm_get_msr()
2933 msr_info->data = svm->vmcb->save.spec_ctrl; in svm_get_msr()
2935 msr_info->data = svm->spec_ctrl; in svm_get_msr()
2938 if (!msr_info->host_initiated && in svm_get_msr()
2942 msr_info->data = svm->virt_spec_ctrl; in svm_get_msr()
2954 msr_info->data = 0; in svm_get_msr()
2958 msr_info->data = 0x1E; in svm_get_msr()
2962 msr_info->data = svm->msr_decfg; in svm_get_msr()
2973 if (!err || !sev_es_guest(vcpu->kvm) || WARN_ON_ONCE(!svm->sev_es.ghcb)) in svm_complete_emulated_msr()
2976 ghcb_set_sw_exit_info_1(svm->sev_es.ghcb, 1); in svm_complete_emulated_msr()
2977 ghcb_set_sw_exit_info_2(svm->sev_es.ghcb, in svm_complete_emulated_msr()
2994 if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK) in svm_set_vm_cr()
2997 svm->nested.vm_cr_msr &= ~chg_mask; in svm_set_vm_cr()
2998 svm->nested.vm_cr_msr |= (data & chg_mask); in svm_set_vm_cr()
3000 svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK; in svm_set_vm_cr()
3003 if (svm_dis && (vcpu->arch.efer & EFER_SVME)) in svm_set_vm_cr()
3014 u32 ecx = msr->index; in svm_set_msr()
3015 u64 data = msr->data; in svm_set_msr()
3018 return vcpu->kvm->arch.has_protected_state ? -EINVAL : 0; in svm_set_msr()
3025 if (!msr->host_initiated) in svm_set_msr()
3035 if (data != 0 && data != svm->tsc_ratio_msr) in svm_set_msr()
3043 svm->tsc_ratio_msr = data; in svm_set_msr()
3055 svm->vmcb01.ptr->save.g_pat = data; in svm_set_msr()
3058 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); in svm_set_msr()
3061 if (!msr->host_initiated && in svm_set_msr()
3069 svm->vmcb->save.spec_ctrl = data; in svm_set_msr()
3071 svm->spec_ctrl = data; in svm_set_msr()
3076 * For non-nested: in svm_set_msr()
3077 * When it's written (to non-zero) for the first time, pass in svm_set_msr()
3086 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1); in svm_set_msr()
3089 if (!msr->host_initiated && in svm_set_msr()
3096 svm->virt_spec_ctrl = data; in svm_set_msr()
3099 svm->vmcb01.ptr->save.star = data; in svm_set_msr()
3103 svm->vmcb01.ptr->save.lstar = data; in svm_set_msr()
3106 svm->vmcb01.ptr->save.cstar = data; in svm_set_msr()
3109 svm->vmcb01.ptr->save.gs.base = data; in svm_set_msr()
3112 svm->vmcb01.ptr->save.fs.base = data; in svm_set_msr()
3115 svm->vmcb01.ptr->save.kernel_gs_base = data; in svm_set_msr()
3118 svm->vmcb01.ptr->save.sfmask = data; in svm_set_msr()
3122 svm->vmcb01.ptr->save.sysenter_cs = data; in svm_set_msr()
3125 svm->vmcb01.ptr->save.sysenter_eip = (u32)data; in svm_set_msr()
3133 svm->sysenter_eip_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; in svm_set_msr()
3136 svm->vmcb01.ptr->save.sysenter_esp = (u32)data; in svm_set_msr()
3137 svm->sysenter_esp_hi = guest_cpuid_is_intel_compatible(vcpu) ? (data >> 32) : 0; in svm_set_msr()
3141 * TSC_AUX is always virtualized for SEV-ES guests when the in svm_set_msr()
3147 if (boot_cpu_has(X86_FEATURE_V_TSC_AUX) && sev_es_guest(vcpu->kvm)) in svm_set_msr()
3153 * guest via direct_access_msrs, and switch it via user return. in svm_set_msr()
3156 ret = kvm_set_user_return_msr(tsc_aux_uret_slot, data, -1ull); in svm_set_msr()
3161 svm->tsc_aux = data; in svm_set_msr()
3170 * AMD changed the architectural behavior of bits 5:2. On CPUs in svm_set_msr()
3171 * without BusLockTrap, bits 5:2 control "external pins", but in svm_set_msr()
3173 * and bits 5:3 are reserved-to-zero. Sadly, old KVM allowed in svm_set_msr()
3174 * the guest to set bits 5:2 despite not actually virtualizing in svm_set_msr()
3175 * Performance-Monitoring/Breakpoint external pins. Drop bits in svm_set_msr()
3182 * way to communicate lack of support to the guest. in svm_set_msr()
3192 svm_get_lbr_vmcb(svm)->save.dbgctl = data; in svm_set_msr()
3202 if (!msr->host_initiated && !page_address_valid(vcpu, data)) in svm_set_msr()
3205 svm->nested.hsave_msr = data & PAGE_MASK; in svm_set_msr()
3221 svm->msr_decfg = data; in svm_set_msr()
3232 if (to_svm(vcpu)->vmcb->control.exit_info_1) in msr_interception()
3246 * requesting the IRQ window and we have to re-enable it. in interrupt_window_interception()
3253 * AVIC still inhibited due to per-cpu AVIC inhibition. in interrupt_window_interception()
3255 kvm_clear_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); in interrupt_window_interception()
3257 ++vcpu->stat.irq_window_exits; in interrupt_window_interception()
3265 * CPL is not made available for an SEV-ES guest, therefore in pause_interception()
3266 * vcpu->arch.preempted_in_kernel can never be true. Just in pause_interception()
3269 in_kernel = !sev_es_guest(vcpu->kvm) && svm_get_cpl(vcpu) == 0; in pause_interception()
3293 type = svm->vmcb->control.exit_info_2; in invpcid_interception()
3294 gva = svm->vmcb->control.exit_info_1; in invpcid_interception()
3378 struct vmcb_control_area *control = &svm->vmcb->control; in dump_vmcb()
3379 struct vmcb_save_area *save = &svm->vmcb->save; in dump_vmcb()
3380 struct vmcb_save_area *save01 = &svm->vmcb01.ptr->save; in dump_vmcb()
3388 svm->current_vmcb->ptr, vcpu->arch.last_vmentry_cpu); in dump_vmcb()
3390 pr_err("%-20s%04x\n", "cr_read:", control->intercepts[INTERCEPT_CR] & 0xffff); in dump_vmcb()
3391 pr_err("%-20s%04x\n", "cr_write:", control->intercepts[INTERCEPT_CR] >> 16); in dump_vmcb()
3392 pr_err("%-20s%04x\n", "dr_read:", control->intercepts[INTERCEPT_DR] & 0xffff); in dump_vmcb()
3393 pr_err("%-20s%04x\n", "dr_write:", control->intercepts[INTERCEPT_DR] >> 16); in dump_vmcb()
3394 pr_err("%-20s%08x\n", "exceptions:", control->intercepts[INTERCEPT_EXCEPTION]); in dump_vmcb()
3395 pr_err("%-20s%08x %08x\n", "intercepts:", in dump_vmcb()
3396 control->intercepts[INTERCEPT_WORD3], in dump_vmcb()
3397 control->intercepts[INTERCEPT_WORD4]); in dump_vmcb()
3398 pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count); in dump_vmcb()
3399 pr_err("%-20s%d\n", "pause filter threshold:", in dump_vmcb()
3400 control->pause_filter_thresh); in dump_vmcb()
3401 pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa); in dump_vmcb()
3402 pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa); in dump_vmcb()
3403 pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset); in dump_vmcb()
3404 pr_err("%-20s%d\n", "asid:", control->asid); in dump_vmcb()
3405 pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl); in dump_vmcb()
3406 pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl); in dump_vmcb()
3407 pr_err("%-20s%08x\n", "int_vector:", control->int_vector); in dump_vmcb()
3408 pr_err("%-20s%08x\n", "int_state:", control->int_state); in dump_vmcb()
3409 pr_err("%-20s%08x\n", "exit_code:", control->exit_code); in dump_vmcb()
3410 pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1); in dump_vmcb()
3411 pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2); in dump_vmcb()
3412 pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info); in dump_vmcb()
3413 pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err); in dump_vmcb()
3414 pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl); in dump_vmcb()
3415 pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3); in dump_vmcb()
3416 pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar); in dump_vmcb()
3417 pr_err("%-20s%016llx\n", "ghcb:", control->ghcb_gpa); in dump_vmcb()
3418 pr_err("%-20s%08x\n", "event_inj:", control->event_inj); in dump_vmcb()
3419 pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err); in dump_vmcb()
3420 pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext); in dump_vmcb()
3421 pr_err("%-20s%016llx\n", "next_rip:", control->next_rip); in dump_vmcb()
3422 pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page); in dump_vmcb()
3423 pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id); in dump_vmcb()
3424 pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id); in dump_vmcb()
3425 pr_err("%-20s%016llx\n", "vmsa_pa:", control->vmsa_pa); in dump_vmcb()
3427 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3429 save->es.selector, save->es.attrib, in dump_vmcb()
3430 save->es.limit, save->es.base); in dump_vmcb()
3431 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3433 save->cs.selector, save->cs.attrib, in dump_vmcb()
3434 save->cs.limit, save->cs.base); in dump_vmcb()
3435 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3437 save->ss.selector, save->ss.attrib, in dump_vmcb()
3438 save->ss.limit, save->ss.base); in dump_vmcb()
3439 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3441 save->ds.selector, save->ds.attrib, in dump_vmcb()
3442 save->ds.limit, save->ds.base); in dump_vmcb()
3443 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3445 save01->fs.selector, save01->fs.attrib, in dump_vmcb()
3446 save01->fs.limit, save01->fs.base); in dump_vmcb()
3447 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3449 save01->gs.selector, save01->gs.attrib, in dump_vmcb()
3450 save01->gs.limit, save01->gs.base); in dump_vmcb()
3451 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3453 save->gdtr.selector, save->gdtr.attrib, in dump_vmcb()
3454 save->gdtr.limit, save->gdtr.base); in dump_vmcb()
3455 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3457 save01->ldtr.selector, save01->ldtr.attrib, in dump_vmcb()
3458 save01->ldtr.limit, save01->ldtr.base); in dump_vmcb()
3459 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3461 save->idtr.selector, save->idtr.attrib, in dump_vmcb()
3462 save->idtr.limit, save->idtr.base); in dump_vmcb()
3463 pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n", in dump_vmcb()
3465 save01->tr.selector, save01->tr.attrib, in dump_vmcb()
3466 save01->tr.limit, save01->tr.base); in dump_vmcb()
3468 save->vmpl, save->cpl, save->efer); in dump_vmcb()
3469 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3470 "cr0:", save->cr0, "cr2:", save->cr2); in dump_vmcb()
3471 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3472 "cr3:", save->cr3, "cr4:", save->cr4); in dump_vmcb()
3473 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3474 "dr6:", save->dr6, "dr7:", save->dr7); in dump_vmcb()
3475 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3476 "rip:", save->rip, "rflags:", save->rflags); in dump_vmcb()
3477 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3478 "rsp:", save->rsp, "rax:", save->rax); in dump_vmcb()
3479 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3480 "star:", save01->star, "lstar:", save01->lstar); in dump_vmcb()
3481 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3482 "cstar:", save01->cstar, "sfmask:", save01->sfmask); in dump_vmcb()
3483 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3484 "kernel_gs_base:", save01->kernel_gs_base, in dump_vmcb()
3485 "sysenter_cs:", save01->sysenter_cs); in dump_vmcb()
3486 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3487 "sysenter_esp:", save01->sysenter_esp, in dump_vmcb()
3488 "sysenter_eip:", save01->sysenter_eip); in dump_vmcb()
3489 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3490 "gpat:", save->g_pat, "dbgctl:", save->dbgctl); in dump_vmcb()
3491 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3492 "br_from:", save->br_from, "br_to:", save->br_to); in dump_vmcb()
3493 pr_err("%-15s %016llx %-13s %016llx\n", in dump_vmcb()
3494 "excp_from:", save->last_excp_from, in dump_vmcb()
3495 "excp_to:", save->last_excp_to); in dump_vmcb()
3508 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR; in svm_handle_invalid_exit()
3509 vcpu->run->internal.suberror = KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON; in svm_handle_invalid_exit()
3510 vcpu->run->internal.ndata = 2; in svm_handle_invalid_exit()
3511 vcpu->run->internal.data[0] = exit_code; in svm_handle_invalid_exit()
3512 vcpu->run->internal.data[1] = vcpu->arch.last_vmentry_cpu; in svm_handle_invalid_exit()
3540 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; in svm_get_exit_info()
3542 *reason = control->exit_code; in svm_get_exit_info()
3543 *info1 = control->exit_info_1; in svm_get_exit_info()
3544 *info2 = control->exit_info_2; in svm_get_exit_info()
3545 *intr_info = control->exit_int_info; in svm_get_exit_info()
3548 *error_code = control->exit_int_info_err; in svm_get_exit_info()
3556 struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control; in svm_get_entry_info()
3558 *intr_info = control->event_inj; in svm_get_entry_info()
3562 *error_code = control->event_inj_err; in svm_get_entry_info()
3571 struct kvm_run *kvm_run = vcpu->run; in svm_handle_exit()
3572 u32 exit_code = svm->vmcb->control.exit_code; in svm_handle_exit()
3574 /* SEV-ES guests must use the CR write traps to track CR registers. */ in svm_handle_exit()
3575 if (!sev_es_guest(vcpu->kvm)) { in svm_handle_exit()
3577 vcpu->arch.cr0 = svm->vmcb->save.cr0; in svm_handle_exit()
3579 vcpu->arch.cr3 = svm->vmcb->save.cr3; in svm_handle_exit()
3596 if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) { in svm_handle_exit()
3597 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY; in svm_handle_exit()
3598 kvm_run->fail_entry.hardware_entry_failure_reason in svm_handle_exit()
3599 = svm->vmcb->control.exit_code; in svm_handle_exit()
3600 kvm_run->fail_entry.cpu = vcpu->arch.last_vmentry_cpu; in svm_handle_exit()
3613 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in pre_svm_run()
3619 * vmcb clean bits are per logical CPU, as are KVM's asid assignments. in pre_svm_run()
3621 if (unlikely(svm->current_vmcb->cpu != vcpu->cpu)) { in pre_svm_run()
3622 svm->current_vmcb->asid_generation = 0; in pre_svm_run()
3623 vmcb_mark_all_dirty(svm->vmcb); in pre_svm_run()
3624 svm->current_vmcb->cpu = vcpu->cpu; in pre_svm_run()
3627 if (sev_guest(vcpu->kvm)) in pre_svm_run()
3628 return pre_sev_run(svm, vcpu->cpu); in pre_svm_run()
3631 if (svm->current_vmcb->asid_generation != sd->asid_generation) in pre_svm_run()
3639 svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI; in svm_inject_nmi()
3641 if (svm->nmi_l1_to_l2) in svm_inject_nmi()
3650 svm->nmi_masked = true; in svm_inject_nmi()
3653 ++vcpu->stat.nmi_injections; in svm_inject_nmi()
3663 return !!(svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK); in svm_is_vnmi_pending()
3673 if (svm->vmcb->control.int_ctl & V_NMI_PENDING_MASK) in svm_set_vnmi_pending()
3676 svm->vmcb->control.int_ctl |= V_NMI_PENDING_MASK; in svm_set_vnmi_pending()
3677 vmcb_mark_dirty(svm->vmcb, VMCB_INTR); in svm_set_vnmi_pending()
3684 ++vcpu->stat.nmi_injections; in svm_set_vnmi_pending()
3694 if (vcpu->arch.interrupt.soft) { in svm_inject_irq()
3703 trace_kvm_inj_virq(vcpu->arch.interrupt.nr, in svm_inject_irq()
3704 vcpu->arch.interrupt.soft, reinjected); in svm_inject_irq()
3705 ++vcpu->stat.irq_injections; in svm_inject_irq()
3707 svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr | in svm_inject_irq()
3715 * apic->apicv_active must be read after vcpu->mode. in svm_complete_interrupt_delivery()
3718 bool in_guest_mode = (smp_load_acquire(&vcpu->mode) == IN_GUEST_MODE); in svm_complete_interrupt_delivery()
3720 /* Note, this is called iff the local APIC is in-kernel. */ in svm_complete_interrupt_delivery()
3721 if (!READ_ONCE(vcpu->arch.apic->apicv_active)) { in svm_complete_interrupt_delivery()
3728 trace_kvm_apicv_accept_irq(vcpu->vcpu_id, delivery_mode, trig_mode, vector); in svm_complete_interrupt_delivery()
3732 * the vCPU exits the guest before the doorbell chimes, hardware in svm_complete_interrupt_delivery()
3751 * Pairs with the smp_mb_*() after setting vcpu->guest_mode in in svm_deliver_interrupt()
3755 * will signal the doorbell if the CPU has already entered the guest. in svm_deliver_interrupt()
3758 svm_complete_interrupt_delivery(apic->vcpu, delivery_mode, trig_mode, vector); in svm_deliver_interrupt()
3766 * SEV-ES guests must always keep the CR intercepts cleared. CR in svm_update_cr8_intercept()
3769 if (sev_es_guest(vcpu->kvm)) in svm_update_cr8_intercept()
3777 if (irr == -1) in svm_update_cr8_intercept()
3789 return svm->vmcb->control.int_ctl & V_NMI_BLOCKING_MASK; in svm_get_nmi_mask()
3791 return svm->nmi_masked; in svm_get_nmi_mask()
3800 svm->vmcb->control.int_ctl |= V_NMI_BLOCKING_MASK; in svm_set_nmi_mask()
3802 svm->vmcb->control.int_ctl &= ~V_NMI_BLOCKING_MASK; in svm_set_nmi_mask()
3805 svm->nmi_masked = masked; in svm_set_nmi_mask()
3816 struct vmcb *vmcb = svm->vmcb; in svm_nmi_blocked()
3827 return vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK; in svm_nmi_blocked()
3833 if (svm->nested.nested_run_pending) in svm_nmi_allowed()
3834 return -EBUSY; in svm_nmi_allowed()
3839 /* An NMI must not be injected into L2 if it's supposed to VM-Exit. */ in svm_nmi_allowed()
3841 return -EBUSY; in svm_nmi_allowed()
3848 struct vmcb *vmcb = svm->vmcb; in svm_interrupt_blocked()
3855 if ((svm->nested.ctl.int_ctl & V_INTR_MASKING_MASK) in svm_interrupt_blocked()
3856 ? !(svm->vmcb01.ptr->save.rflags & X86_EFLAGS_IF) in svm_interrupt_blocked()
3868 return (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK); in svm_interrupt_blocked()
3875 if (svm->nested.nested_run_pending) in svm_interrupt_allowed()
3876 return -EBUSY; in svm_interrupt_allowed()
3882 * An IRQ must not be injected into L2 if it's supposed to VM-Exit, in svm_interrupt_allowed()
3886 return -EBUSY; in svm_interrupt_allowed()
3915 kvm_set_apicv_inhibit(vcpu->kvm, APICV_INHIBIT_REASON_IRQWIN); in svm_enable_irq_window()
3929 * If KVM has already intercepted IRET, then single-step over the IRET, in svm_enable_nmi_window()
3933 * are masked, as KVM allows at most one to-be-injected NMI and one in svm_enable_nmi_window()
3938 * inject the NMI. In those situations, KVM needs to single-step over in svm_enable_nmi_window()
3944 if (!svm->awaiting_iret_completion) in svm_enable_nmi_window()
3949 * SEV-ES guests are responsible for signaling when a vCPU is ready to in svm_enable_nmi_window()
3950 * receive a new NMI, as SEV-ES guests can't be single-stepped, i.e. in svm_enable_nmi_window()
3951 * KVM can't intercept and single-step IRET to detect when NMIs are in svm_enable_nmi_window()
3954 * Note, GIF is guaranteed to be '1' for SEV-ES guests as hardware in svm_enable_nmi_window()
3955 * ignores SEV-ES guest writes to EFER.SVME *and* CLGI/STGI are not in svm_enable_nmi_window()
3958 if (sev_es_guest(vcpu->kvm)) in svm_enable_nmi_window()
3971 svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu); in svm_enable_nmi_window()
3972 svm->nmi_singlestep = true; in svm_enable_nmi_window()
3973 svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF); in svm_enable_nmi_window()
3982 * A TLB flush for the current ASID flushes both "host" and "guest" TLB in svm_flush_tlb_asid()
3983 * entries, and thus is a superset of Hyper-V's fine grained flushing. in svm_flush_tlb_asid()
3991 * unconditionally does a TLB flush on both nested VM-Enter and nested in svm_flush_tlb_asid()
3992 * VM-Exit (via kvm_mmu_reset_context()). in svm_flush_tlb_asid()
3995 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID; in svm_flush_tlb_asid()
3997 svm->current_vmcb->asid_generation--; in svm_flush_tlb_asid()
4002 hpa_t root_tdp = vcpu->arch.mmu->root.hpa; in svm_flush_tlb_current()
4005 * When running on Hyper-V with EnlightenedNptTlb enabled, explicitly in svm_flush_tlb_current()
4007 * affects virtual to physical mappings, it does not invalidate guest in svm_flush_tlb_current()
4019 * When running on Hyper-V with EnlightenedNptTlb enabled, remote TLB in svm_flush_tlb_all()
4023 * which might be fatal to the guest. Yell, but try to recover. in svm_flush_tlb_all()
4026 hv_flush_remote_tlbs(vcpu->kvm); in svm_flush_tlb_all()
4035 invlpga(gva, svm->vmcb->control.asid); in svm_flush_tlb_gva()
4046 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK; in sync_cr8_to_lapic()
4061 svm->vmcb->control.int_ctl &= ~V_TPR_MASK; in sync_lapic_to_cr8()
4062 svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK; in sync_lapic_to_cr8()
4073 * If NRIPS is enabled, KVM must snapshot the pre-VMRUN next_rip that's in svm_complete_soft_interrupt()
4076 * needs to manually set next_rip for re-injection. Unlike the !nrips in svm_complete_soft_interrupt()
4077 * case below, this needs to be done if and only if KVM is re-injecting in svm_complete_soft_interrupt()
4082 kvm_is_linear_rip(vcpu, svm->soft_int_old_rip + svm->soft_int_csbase)) in svm_complete_soft_interrupt()
4083 svm->vmcb->control.next_rip = svm->soft_int_next_rip; in svm_complete_soft_interrupt()
4089 * hit a #NP in the guest, and the #NP encountered a #PF, the #NP will in svm_complete_soft_interrupt()
4093 kvm_is_linear_rip(vcpu, svm->soft_int_next_rip + svm->soft_int_csbase)) in svm_complete_soft_interrupt()
4094 kvm_rip_write(vcpu, svm->soft_int_old_rip); in svm_complete_soft_interrupt()
4102 u32 exitintinfo = svm->vmcb->control.exit_int_info; in svm_complete_interrupts()
4103 bool nmi_l1_to_l2 = svm->nmi_l1_to_l2; in svm_complete_interrupts()
4104 bool soft_int_injected = svm->soft_int_injected; in svm_complete_interrupts()
4106 svm->nmi_l1_to_l2 = false; in svm_complete_interrupts()
4107 svm->soft_int_injected = false; in svm_complete_interrupts()
4113 if (svm->awaiting_iret_completion && in svm_complete_interrupts()
4114 kvm_rip_read(vcpu) != svm->nmi_iret_rip) { in svm_complete_interrupts()
4115 svm->awaiting_iret_completion = false; in svm_complete_interrupts()
4116 svm->nmi_masked = false; in svm_complete_interrupts()
4120 vcpu->arch.nmi_injected = false; in svm_complete_interrupts()
4137 vcpu->arch.nmi_injected = true; in svm_complete_interrupts()
4138 svm->nmi_l1_to_l2 = nmi_l1_to_l2; in svm_complete_interrupts()
4142 * Never re-inject a #VC exception. in svm_complete_interrupts()
4148 u32 err = svm->vmcb->control.exit_int_info_err; in svm_complete_interrupts()
4169 struct vmcb_control_area *control = &svm->vmcb->control; in svm_cancel_injection()
4171 control->exit_int_info = control->event_inj; in svm_cancel_injection()
4172 control->exit_int_info_err = control->event_inj_err; in svm_cancel_injection()
4173 control->event_inj = 0; in svm_cancel_injection()
4179 if (to_kvm_sev_info(vcpu->kvm)->need_init) in svm_vcpu_pre_run()
4180 return -EINVAL; in svm_vcpu_pre_run()
4192 switch (svm->vmcb->control.exit_code) { in svm_exit_handlers_fastpath()
4194 if (!svm->vmcb->control.exit_info_1) in svm_exit_handlers_fastpath()
4208 struct svm_cpu_data *sd = per_cpu_ptr(&svm_data, vcpu->cpu); in svm_vcpu_enter_exit()
4218 * into guest state if delivery of an event during VMRUN triggers a in svm_vcpu_enter_exit()
4227 if (sev_es_guest(vcpu->kvm)) in svm_vcpu_enter_exit()
4246 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; in svm_vcpu_run()
4247 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; in svm_vcpu_run()
4248 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; in svm_vcpu_run()
4256 if (svm->nmi_singlestep && svm->vmcb->control.event_inj) { in svm_vcpu_run()
4267 smp_send_reschedule(vcpu->cpu); in svm_vcpu_run()
4273 if (unlikely(svm->asid != svm->vmcb->control.asid)) { in svm_vcpu_run()
4274 svm->vmcb->control.asid = svm->asid; in svm_vcpu_run()
4275 vmcb_mark_dirty(svm->vmcb, VMCB_ASID); in svm_vcpu_run()
4277 svm->vmcb->save.cr2 = vcpu->arch.cr2; in svm_vcpu_run()
4279 svm_hv_update_vp_id(svm->vmcb, vcpu); in svm_vcpu_run()
4282 * Run with all-zero DR6 unless needed, so that we can get the exact cause in svm_vcpu_run()
4285 if (likely(!(vcpu->arch.switch_db_regs & KVM_DEBUGREG_WONT_EXIT))) in svm_vcpu_run()
4294 * VM-Exit), as running with the host's DEBUGCTL can negatively affect in svm_vcpu_run()
4295 * guest state and can even be fatal, e.g. due to Bus Lock Detect. in svm_vcpu_run()
4297 if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) && in svm_vcpu_run()
4298 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) in svm_vcpu_run()
4299 update_debugctlmsr(svm->vmcb->save.dbgctl); in svm_vcpu_run()
4304 * If this vCPU has touched SPEC_CTRL, restore the guest's value if in svm_vcpu_run()
4305 * it's non-zero. Since vmentry is serialising on affected CPUs, there in svm_vcpu_run()
4310 x86_spec_ctrl_set_guest(svm->virt_spec_ctrl); in svm_vcpu_run()
4315 x86_spec_ctrl_restore_host(svm->virt_spec_ctrl); in svm_vcpu_run()
4317 if (!sev_es_guest(vcpu->kvm)) { in svm_vcpu_run()
4318 vcpu->arch.cr2 = svm->vmcb->save.cr2; in svm_vcpu_run()
4319 vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax; in svm_vcpu_run()
4320 vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp; in svm_vcpu_run()
4321 vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip; in svm_vcpu_run()
4323 vcpu->arch.regs_dirty = 0; in svm_vcpu_run()
4325 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) in svm_vcpu_run()
4328 if (!(svm->vmcb->control.virt_ext & LBR_CTL_ENABLE_MASK) && in svm_vcpu_run()
4329 vcpu->arch.host_debugctl != svm->vmcb->save.dbgctl) in svm_vcpu_run()
4330 update_debugctlmsr(vcpu->arch.host_debugctl); in svm_vcpu_run()
4337 if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI)) in svm_vcpu_run()
4342 svm->next_rip = 0; in svm_vcpu_run()
4347 if (svm->nested.nested_run_pending && in svm_vcpu_run()
4348 svm->vmcb->control.exit_code != SVM_EXIT_ERR) in svm_vcpu_run()
4349 ++vcpu->stat.nested_run; in svm_vcpu_run()
4351 svm->nested.nested_run_pending = 0; in svm_vcpu_run()
4354 svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING; in svm_vcpu_run()
4355 vmcb_mark_all_clean(svm->vmcb); in svm_vcpu_run()
4358 if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) in svm_vcpu_run()
4359 vcpu->arch.apf.host_apf_flags = in svm_vcpu_run()
4362 vcpu->arch.regs_avail &= ~SVM_REGS_LAZY_LOAD_SET; in svm_vcpu_run()
4368 if (unlikely(svm->vmcb->control.exit_code == in svm_vcpu_run()
4386 svm->vmcb->control.nested_cr3 = __sme_set(root_hpa); in svm_load_mmu_pgd()
4387 vmcb_mark_dirty(svm->vmcb, VMCB_NPT); in svm_load_mmu_pgd()
4391 cr3 = vcpu->arch.cr3; in svm_load_mmu_pgd()
4395 /* PCID in the guest should be impossible with a 32-bit MMU. */ in svm_load_mmu_pgd()
4400 svm->vmcb->save.cr3 = cr3; in svm_load_mmu_pgd()
4401 vmcb_mark_dirty(svm->vmcb, VMCB_CR); in svm_load_mmu_pgd()
4419 static bool svm_has_emulated_msr(struct kvm *kvm, u32 index) in svm_has_emulated_msr() argument
4421 switch (index) { in svm_has_emulated_msr()
4428 /* SEV-ES guests do not support SMM, so report false */ in svm_has_emulated_msr()
4444 * SVM doesn't provide a way to disable just XSAVES in the guest, KVM in svm_vcpu_after_set_cpuid()
4447 * guest has XSAVE enabled, the guest can execute XSAVES without in svm_vcpu_after_set_cpuid()
4449 * whether it's advertised to the guest so that KVM context switches in svm_vcpu_after_set_cpuid()
4450 * XSS on VM-Enter/VM-Exit. Failure to do so would effectively give in svm_vcpu_after_set_cpuid()
4451 * the guest read/write access to the host's XSS. in svm_vcpu_after_set_cpuid()
4459 * VMLOAD drops bits 63:32 of SYSENTER (ignoring the fact that exposing in svm_vcpu_after_set_cpuid()
4468 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_PRED_CMD, 0, in svm_vcpu_after_set_cpuid()
4472 set_msr_interception(vcpu, svm->msrpm, MSR_IA32_FLUSH_CMD, 0, in svm_vcpu_after_set_cpuid()
4475 if (sev_guest(vcpu->kvm)) in svm_vcpu_after_set_cpuid()
4558 struct vmcb *vmcb = svm->vmcb; in svm_check_intercept()
4560 if (info->intercept >= ARRAY_SIZE(x86_intercept_map)) in svm_check_intercept()
4563 icpt_info = x86_intercept_map[info->intercept]; in svm_check_intercept()
4570 if (info->intercept == x86_intercept_cr_read) in svm_check_intercept()
4571 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4576 if (info->intercept == x86_intercept_cr_write) in svm_check_intercept()
4577 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4580 info->intercept == x86_intercept_clts) in svm_check_intercept()
4583 if (!(vmcb12_is_intercept(&svm->nested.ctl, in svm_check_intercept()
4587 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK; in svm_check_intercept()
4588 val = info->src_val & ~SVM_CR0_SELECTIVE_MASK; in svm_check_intercept()
4590 if (info->intercept == x86_intercept_lmsw) { in svm_check_intercept()
4593 /* lmsw can't clear PE - catch this here */ in svm_check_intercept()
4605 icpt_info.exit_code += info->modrm_reg; in svm_check_intercept()
4608 if (info->intercept == x86_intercept_wrmsr) in svm_check_intercept()
4609 vmcb->control.exit_info_1 = 1; in svm_check_intercept()
4611 vmcb->control.exit_info_1 = 0; in svm_check_intercept()
4618 if (info->rep_prefix != REPE_PREFIX) in svm_check_intercept()
4625 if (info->intercept == x86_intercept_in || in svm_check_intercept()
4626 info->intercept == x86_intercept_ins) { in svm_check_intercept()
4627 exit_info = ((info->src_val & 0xffff) << 16) | in svm_check_intercept()
4629 bytes = info->dst_bytes; in svm_check_intercept()
4631 exit_info = (info->dst_val & 0xffff) << 16; in svm_check_intercept()
4632 bytes = info->src_bytes; in svm_check_intercept()
4635 if (info->intercept == x86_intercept_outs || in svm_check_intercept()
4636 info->intercept == x86_intercept_ins) in svm_check_intercept()
4639 if (info->rep_prefix) in svm_check_intercept()
4646 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1); in svm_check_intercept()
4648 vmcb->control.exit_info_1 = exit_info; in svm_check_intercept()
4649 vmcb->control.exit_info_2 = info->next_rip; in svm_check_intercept()
4657 /* TODO: Advertise NRIPS to guest hypervisor unconditionally */ in svm_check_intercept()
4659 vmcb->control.next_rip = info->next_rip; in svm_check_intercept()
4660 vmcb->control.exit_code = icpt_info.exit_code; in svm_check_intercept()
4672 if (to_svm(vcpu)->vmcb->control.exit_code == SVM_EXIT_INTR) in svm_handle_exit_irqoff()
4673 vcpu->arch.at_instruction_boundary = true; in svm_handle_exit_irqoff()
4679 vcpu->arch.mcg_cap &= 0x1ff; in svm_setup_mce()
4697 if (svm->nested.nested_run_pending) in svm_smi_allowed()
4698 return -EBUSY; in svm_smi_allowed()
4703 /* An SMI must not be injected into L2 if it's supposed to VM-Exit. */ in svm_smi_allowed()
4705 return -EBUSY; in svm_smi_allowed()
4720 * 32-bit SMRAM format doesn't preserve EFER and SVM state. Userspace is in svm_enter_smm()
4727 smram->smram64.svm_guest_flag = 1; in svm_enter_smm()
4728 smram->smram64.svm_guest_vmcb_gpa = svm->nested.vmcb12_gpa; in svm_enter_smm()
4730 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX]; in svm_enter_smm()
4731 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP]; in svm_enter_smm()
4732 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP]; in svm_enter_smm()
4741 * be lost. Temporary save non-VMLOAD/VMSAVE state to the host save in svm_enter_smm()
4743 * format of the area is identical to guest save area offsetted in svm_enter_smm()
4750 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) in svm_enter_smm()
4756 &svm->vmcb01.ptr->save); in svm_enter_smm()
4769 const struct kvm_smram_state_64 *smram64 = &smram->smram64; in svm_leave_smm()
4774 /* Non-zero if SMI arrived while vCPU was in guest mode. */ in svm_leave_smm()
4775 if (!smram64->svm_guest_flag) in svm_leave_smm()
4781 if (!(smram64->efer & EFER_SVME)) in svm_leave_smm()
4784 if (kvm_vcpu_map(vcpu, gpa_to_gfn(smram64->svm_guest_vmcb_gpa), &map)) in svm_leave_smm()
4788 if (kvm_vcpu_map(vcpu, gpa_to_gfn(svm->nested.hsave_msr), &map_save)) in svm_leave_smm()
4799 svm_copy_vmrun_state(&svm->vmcb01.ptr->save, map_save.hva + 0x400); in svm_leave_smm()
4802 * Enter the nested guest now in svm_leave_smm()
4805 vmcb_mark_all_dirty(svm->vmcb01.ptr); in svm_leave_smm()
4808 nested_copy_vmcb_control_to_cache(svm, &vmcb12->control); in svm_leave_smm()
4809 nested_copy_vmcb_save_to_cache(svm, &vmcb12->save); in svm_leave_smm()
4810 ret = enter_svm_guest_mode(vcpu, smram64->svm_guest_vmcb_gpa, vmcb12, false); in svm_leave_smm()
4815 svm->nested.nested_run_pending = 1; in svm_leave_smm()
4846 if ((svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK) && in svm_check_emulate_instruction()
4850 /* Emulation is always possible when KVM has access to all guest state. */ in svm_check_emulate_instruction()
4851 if (!sev_guest(vcpu->kvm)) in svm_check_emulate_instruction()
4860 * Emulation is impossible for SEV-ES guests as KVM doesn't have access in svm_check_emulate_instruction()
4861 * to guest register state. in svm_check_emulate_instruction()
4863 if (sev_es_guest(vcpu->kvm)) in svm_check_emulate_instruction()
4876 * available. SEV guest memory is encrypted with a guest specific key in svm_check_emulate_instruction()
4882 * this path should never be hit by a well-behaved guest, e.g. KVM in svm_check_emulate_instruction()
4884 * theoretically reachable, e.g. via unaccelerated fault-like AVIC in svm_check_emulate_instruction()
4885 * access, and needs to be handled by KVM to avoid putting the guest in svm_check_emulate_instruction()
4887 * its the least awful option given lack of insight into the guest. in svm_check_emulate_instruction()
4889 * If KVM is trying to skip an instruction, simply resume the guest. in svm_check_emulate_instruction()
4890 * If a #NPF occurs while the guest is vectoring an INT3/INTO, then KVM in svm_check_emulate_instruction()
4891 * will attempt to re-inject the INT3/INTO and skip the instruction. in svm_check_emulate_instruction()
4892 * In that scenario, retrying the INT3/INTO and hoping the guest will in svm_check_emulate_instruction()
4908 * the guest attempted to fetch from emulated MMIO or a guest page in svm_check_emulate_instruction()
4918 * When CPU raises #NPF on guest data access and vCPU CR4.SMAP=1, it is in svm_check_emulate_instruction()
4920 * read guest memory at CS:RIP and vmcb.GuestIntrBytes will incorrectly in svm_check_emulate_instruction()
4925 * As above, KVM reaches this point iff the VM is an SEV guest, the CPU in svm_check_emulate_instruction()
4932 * #PF, e.g. if the guest attempt to execute from emulated MMIO and in svm_check_emulate_instruction()
4933 * encountered a reserved/not-present #PF. in svm_check_emulate_instruction()
4938 * have been hit as the guest would have encountered a SMEP in svm_check_emulate_instruction()
4944 * while translating guest page tables (see below). in svm_check_emulate_instruction()
4946 error_code = svm->vmcb->control.exit_info_1; in svm_check_emulate_instruction()
4954 pr_err_ratelimited("SEV Guest triggered AMD Erratum 1096\n"); in svm_check_emulate_instruction()
4958 * to avoid killing the guest and to hopefully avoid confusing in svm_check_emulate_instruction()
4959 * the guest kernel too much, e.g. injecting #PF would not be in svm_check_emulate_instruction()
4960 * coherent with respect to the guest's page tables. Request in svm_check_emulate_instruction()
4962 * no fault that KVM can inject without confusing the guest. in svm_check_emulate_instruction()
4975 * If the erratum was not hit, simply resume the guest and let it fault in svm_check_emulate_instruction()
4978 * userspace will kill the guest, and letting the emulator read garbage in svm_check_emulate_instruction()
4979 * will yield random behavior and potentially corrupt the guest. in svm_check_emulate_instruction()
4981 * Simply resuming the guest is technically not a violation of the SEV in svm_check_emulate_instruction()
4983 * accesses for SEV guest are encrypted, regardless of the C-Bit. The in svm_check_emulate_instruction()
4986 * the guest spin is technically "ignoring" the access. in svm_check_emulate_instruction()
5000 if (!sev_es_guest(vcpu->kvm)) in svm_vcpu_deliver_sipi_vector()
5014 int type = kvm->arch.vm_type; in svm_vm_init()
5018 kvm->arch.has_protected_state = in svm_vm_init()
5020 to_kvm_sev_info(kvm)->need_init = true; in svm_vm_init()
5022 kvm->arch.has_private_mem = (type == KVM_X86_SNP_VM); in svm_vm_init()
5023 kvm->arch.pre_fault_allowed = !kvm->arch.has_private_mem; in svm_vm_init()
5027 kvm->arch.pause_in_guest = true; in svm_vm_init()
5218 * If the mask bit location is below 52, then some bits above the in svm_adjust_mmio_mask()
5305 /* Don't advertise Bus Lock Detect to guest if SVM support is absent */ in svm_set_cpu_caps()
5323 return -EOPNOTSUPP; in svm_hardware_setup()
5330 return -ENOMEM; in svm_hardware_setup()
5374 * KVM's MMU doesn't support using 2-level paging for itself, and thus in svm_hardware_setup()
5375 * NPT isn't supported if the host is using 2-level paging since host in svm_hardware_setup()
5502 return -EOPNOTSUPP; in svm_init()