1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (C) 2021-2022 Intel Corporation */
14 #include <asm/ia32.h>
16 #include <asm/insn-eval.h>
31 #define VE_IS_IO_IN(e)		((e) & BIT(3))
37 #define TDCALL_RETURN_CODE(a)	((a) >> 32)  argument
79 /* Read TD-scoped metadata */
93 /* Write TD-scoped metadata */
106  * tdx_mcall_get_report0() - Wrapper to get TDREPORT0 (a.k.a. TDREPORT
108  * @reportdata: Address of the input buffer which contains user-defined
116  * Return 0 on success, -EINVAL for invalid operands, or -EIO on
131 			return -EINVAL;  in tdx_mcall_get_report0()
132 		return -EIO;  in tdx_mcall_get_report0()
140  * tdx_hcall_get_quote() - Wrapper to request TD Quote using GetQuote
145  * @size: size of the tdquote buffer (4KB-aligned).
155 	/* Since buf is a shared memory, set the shared (decrypted) bits */  in tdx_hcall_get_quote()
197  * that no #VE will be delivered for accesses to TD-private memory.
250  * TDX 1.0 generates a #VE when accessing topology-related CPUID leafs (0xB and
252  * In practice, this means that the kernel can only boot with a plain topology.
256  * Enabling the feature  eliminates topology-related #VEs: the TDX module
265 	/* Has the VMM provided a valid topology configuration? */  in enable_cpu_topology_enumeration()
299 	 * Guest-Host-Communication Interface (GHCI), section 2.4.2 TDCALL  in tdx_setup()
305 	 * The highest bit of a guest physical address is the "sharing" bit.  in tdx_setup()
312 	*cc_mask = BIT_ULL(gpa_width - 1);  in tdx_setup()
317 	tdg_vm_wr(TDCS_NOTIFY_ENABLES, 0, -1ULL);  in tdx_setup()
325  * The TDX module spec states that #VE may be injected for a limited set of
328  *  - Emulation of the architectural #VE injection on EPT violation;
330  *  - As a result of guest TD execution of a disallowed instruction,
331  *    a disallowed MSR access, or CPUID virtualization;
333  *  - A notification to the guest TD about anomalous behavior;
335  * The last one is opt-in and is not used by the kernel.
347 	switch (ve->exit_reason) {  in ve_instr_len()
353 		/* It is safe to use ve->instr_len for #VE due instructions */  in ve_instr_len()
354 		return ve->instr_len;  in ve_instr_len()
357 		 * For EPT violations, ve->insn_len is not defined. For those,  in ve_instr_len()
361 		WARN_ONCE(1, "ve->instr_len is not defined for EPT violations");  in ve_instr_len()
364 		WARN_ONCE(1, "Unexpected #VE-type: %lld\n", ve->exit_reason);  in ve_instr_len()
365 		return ve->instr_len;  in ve_instr_len()
379 	 * can be found in TDX Guest-Host-Communication Interface  in __halt()
397 		return -EIO;  in handle_halt()
428 		.r12 = regs->cx,  in read_msr()
433 	 * can be found in TDX Guest-Host-Communication Interface  in read_msr()
437 		return -EIO;  in read_msr()
439 	regs->ax = lower_32_bits(args.r11);  in read_msr()
440 	regs->dx = upper_32_bits(args.r11);  in read_msr()
449 		.r12 = regs->cx,  in write_msr()
450 		.r13 = (u64)regs->dx << 32 | regs->ax,  in write_msr()
455 	 * can be found in TDX Guest-Host-Communication Interface  in write_msr()
459 		return -EIO;  in write_msr()
469 		.r12 = regs->ax,  in handle_cpuid()
470 		.r13 = regs->cx,  in handle_cpuid()
477 	 * Return all-zeros for any CPUID outside the range. It matches CPU  in handle_cpuid()
478 	 * behaviour for non-supported leaf.  in handle_cpuid()
480 	if (regs->ax < 0x40000000 || regs->ax > 0x4FFFFFFF) {  in handle_cpuid()
481 		regs->ax = regs->bx = regs->cx = regs->dx = 0;  in handle_cpuid()
486 	 * Emulate the CPUID instruction via a hypercall. More info about  in handle_cpuid()
487 	 * ABI can be found in TDX Guest-Host-Communication Interface  in handle_cpuid()
491 		return -EIO;  in handle_cpuid()
494 	 * As per TDX GHCI CPUID ABI, r12-r15 registers contain contents of  in handle_cpuid()
498 	regs->ax = args.r12;  in handle_cpuid()
499 	regs->bx = args.r13;  in handle_cpuid()
500 	regs->cx = args.r14;  in handle_cpuid()
501 	regs->dx = args.r15;  in handle_cpuid()
538 	/* Only in-kernel MMIO is supported */  in handle_mmio()
540 		return -EFAULT;  in handle_mmio()
542 	if (copy_from_kernel_nofault(buffer, (void *)regs->ip, MAX_INSN_SIZE))  in handle_mmio()
543 		return -EFAULT;  in handle_mmio()
546 		return -EINVAL;  in handle_mmio()
550 		return -EINVAL;  in handle_mmio()
555 			return -EINVAL;  in handle_mmio()
558 	if (!fault_in_kernel_space(ve->gla)) {  in handle_mmio()
560 		return -EINVAL;  in handle_mmio()
568 	 * a bug or a load_unaligned_zeropad() that stepped into an MMIO page.  in handle_mmio()
573 	if (vaddr / PAGE_SIZE != (vaddr + size - 1) / PAGE_SIZE)  in handle_mmio()
574 		return -EFAULT;  in handle_mmio()
580 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
581 			return -EIO;  in handle_mmio()
585 		if (!mmio_write(size, ve->gpa, val))  in handle_mmio()
586 			return -EIO;  in handle_mmio()
600 		return -EINVAL;  in handle_mmio()
603 		return -EINVAL;  in handle_mmio()
607 	if (!mmio_read(size, ve->gpa, &val))  in handle_mmio()
608 		return -EIO;  in handle_mmio()
612 		/* Zero-extend for 32-bit operation */  in handle_mmio()
630 		return -EINVAL;  in handle_mmio()
653 	 * in TDX Guest-Host-Communication Interface (GHCI) section titled  in handle_in()
659 	regs->ax &= ~mask;  in handle_in()
661 		regs->ax |= args.r11 & mask;  in handle_in()
672 	 * in TDX Guest-Host-Communication Interface (GHCI) section titled  in handle_out()
676 			       PORT_WRITE, port, regs->ax & mask);  in handle_out()
689 	u32 exit_qual = ve->exit_qual;  in handle_io()
694 		return -EIO;  in handle_io()
706 		return -EIO;  in handle_io()
712  * Early #VE exception handler. Only handles a subset of port I/O.
729 	regs->ip += insn_len;  in tdx_early_handle_ve()
741 	 * This has to be called early in #VE handling.  A "nested" #VE which  in tdx_get_ve_info()
742 	 * occurs before this will raise a #DF and is not recoverable.  in tdx_get_ve_info()
750 	 * valid flag is set. It means that NMI=>#VE will not result in a #DF.  in tdx_get_ve_info()
755 	ve->exit_reason = args.rcx;  in tdx_get_ve_info()
756 	ve->exit_qual   = args.rdx;  in tdx_get_ve_info()
757 	ve->gla         = args.r8;  in tdx_get_ve_info()
758 	ve->gpa         = args.r9;  in tdx_get_ve_info()
759 	ve->instr_len   = lower_32_bits(args.r10);  in tdx_get_ve_info()
760 	ve->instr_info  = upper_32_bits(args.r10);  in tdx_get_ve_info()
767  * or -errno on error.
771 	switch (ve->exit_reason) {  in virt_exception_user()
775 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_user()
776 		return -EIO;  in virt_exception_user()
789  * or -errno on error.
793 	switch (ve->exit_reason) {  in virt_exception_kernel()
803 		if (is_private_gpa(ve->gpa))  in virt_exception_kernel()
804 			panic("Unexpected EPT-violation on private memory.");  in virt_exception_kernel()
809 		pr_warn("Unexpected #VE: %lld\n", ve->exit_reason);  in virt_exception_kernel()
810 		return -EIO;  in virt_exception_kernel()
826 	regs->ip += insn_len;  in tdx_handle_virt_exception()
834 	 * TDX guest is responsible for flushing TLB on private->shared  in tdx_tlb_flush_required()
835 	 * transition. VMM is responsible for flushing on shared->private.  in tdx_tlb_flush_required()
843 	 * flush to avoid integrity failures in the face of a buggy or  in tdx_tlb_flush_required()
862  * can be found in TDX Guest-Host-Communication Interface (GHCI),
867 	/* Retrying the hypercall a second time should succeed; use 3 just in case */  in tdx_map_gpa()
868 	const int max_retries_per_page = 3;  in tdx_map_gpa()
882 			.r13 = end - start };  in tdx_map_gpa()
898 		/* "Consume" a retry without forward progress */  in tdx_map_gpa()
924 	/* shared->private conversion requires memory to be accepted before use */  in tdx_enc_status_changed()
935 	 * Only handle shared->private conversion here.  in tdx_enc_status_change_prepare()
939 		return -EIO;  in tdx_enc_status_change_prepare()
948 	 * Only handle private->shared conversion here.  in tdx_enc_status_change_finish()
952 		return -EIO;  in tdx_enc_status_change_finish()
962 /* Stop new private<->shared conversions */
975 		pr_warn("Failed to stop shared<->private conversions\n");  in tdx_kexec_begin()
1018 			 * *next* kernel through a private mapping will result  in tdx_kexec_finish()
1022 			 * a pre-reserved memory range that is always private.  in tdx_kexec_finish()
1024 			 * a crash if it accesses unconverted memory through  in tdx_kexec_finish()
1025 			 * a private mapping which is possible when accessing  in tdx_kexec_finish()
1032 				pr_err("Failed to unshare range %#lx-%#lx\n",  in tdx_kexec_finish()
1068 	u32 eax, sig[3];  in tdx_early_init()
1093 	physical_mask &= cc_mask - 1;  in tdx_early_init()
1101 	 *   - Private mapping => Shared Page  == Guest shutdown  in tdx_early_init()
1102          *   - Shared mapping  => Private Page == Recoverable #VE  in tdx_early_init()
1110 	 * In both cases there is a temporary shared mapping to a private page,  in tdx_early_init()
1111 	 * which can result in a #VE.  But, there is never a private mapping to  in tdx_early_init()
1112 	 * a shared page.  in tdx_early_init()
1124 	 * Avoid "sti;hlt" execution in TDX guests as HLT induces a #VE that  in tdx_early_init()
1126 	 * in STI-shadow, possibly resulting in missed wakeup events.  in tdx_early_init()
1131 	 * in addition to having a reliable halt logic execution.  in tdx_early_init()
1141 	 * Intel-TDX has a secure RDMSR hypercall, but that needs to be  in tdx_early_init()