1 // SPDX-License-Identifier: GPL-2.0-only
12  *   Signed-off-by: Richard Fellner <[email protected]>
13  *   Signed-off-by: Moritz Lipp <[email protected]>
14  *   Signed-off-by: Daniel Gruss <[email protected]>
15  *   Signed-off-by: Michael Schwarz <[email protected]>
43 #define pr_fmt(fmt)     "Kernel/User page tables isolation: " fmt
51  * Define the page-table levels we clone for user-space on 32
112 		return -EINVAL;  in pti_parse_cmdline()
132 	 * Top-level entries added to init_mm's usermode pgd after boot  in __pti_set_user_pgtbl()
139 	 * The user page tables get the full PGD, accessible from  in __pti_set_user_pgtbl()
142 	kernel_to_user_pgdp(pgdp)->pgd = pgd.pgd;  in __pti_set_user_pgtbl()
145 	 * If this is normal user memory, make it NX in the kernel  in __pti_set_user_pgtbl()
148 	 * instead of allowing user code to execute with the wrong CR3.  in __pti_set_user_pgtbl()
151 	 *  - _PAGE_USER is not set.  This could be an executable  in __pti_set_user_pgtbl()
154 	 *  - we don't have NX support  in __pti_set_user_pgtbl()
155 	 *  - we're clearing the PGD (i.e. the new pgd is not present).  in __pti_set_user_pgtbl()
166  * Walk the user copy of the page tables (optionally) trying to allocate
177 		WARN_ONCE(1, "attempt to walk user address\n");  in pti_user_pagetable_walk_p4d()
194  * Walk the user copy of the page tables (optionally) trying to allocate
219 	/* The user page tables do not use large mappings: */  in pti_user_pagetable_walk_pmd()
240  * user/shadow page tables.  It is never used for userspace data.
275 		WARN_ONCE(1, "attempt to walk to user pte\n");  in pti_user_pagetable_walk_pte()
285 	unsigned int level;  in pti_setup_vsyscall()  local
287 	pte = lookup_address(VSYSCALL_ADDR, &level);  in pti_setup_vsyscall()
288 	if (!pte || WARN_ON(level != PG_LEVEL_4K) || pte_none(*pte))  in pti_setup_vsyscall()
309 		  enum pti_clone_level level, bool late_text)  in pti_clone_pgtable()  argument
349 		if (pmd_leaf(*pmd) || level == PTI_CLONE_PMD) {  in pti_clone_pgtable()
357 			 * called on well-known addresses anyway, so a non-  in pti_clone_pgtable()
365 			 * the user and kernel page tables.  It is effectively  in pti_clone_pgtable()
377 			 * tables will share the last-level page tables of this  in pti_clone_pgtable()
384 		} else if (level == PTI_CLONE_PTE) {  in pti_clone_pgtable()
386 			/* Walk the page-table down to the pte level */  in pti_clone_pgtable()
397 			/* Allocate PTE in the user page-table */  in pti_clone_pgtable()
419  * Clone a single p4d (i.e. a top-level entry on 4-level systems and a
420  * next-level entry on 5-level systems.
437  * Clone the CPU_ENTRY_AREA and associated data into the user space visible
473  * address space into the user page-tables, making PTI useless. So clone
474  * the page-table on the PMD level to prevent that.
488  * Clone the ESPFIX P4D into the user space visible page table
511  * to Meltdown-style attacks which make it trivial to find gadgets or
544 	 * data structures.  Keep the kernel image non-global in  in pti_kernel_image_global_ok()
555  * For some configurations, map all of kernel text into the user page
556  * tables.  This reduces TLB misses, especially on non-PCID systems.
572 	pr_debug("mapping partial kernel image into user address space\n");  in pti_clone_kernel_text()
584 	 * the last level for areas that are not huge-page-aligned.  in pti_clone_kernel_text()
587 	/* Set the global bit for normal non-__init kernel text: */  in pti_clone_kernel_text()
588 	set_memory_global(start, (end_global - start) >> PAGE_SHIFT);  in pti_clone_kernel_text()
607 	set_memory_nonglobal(start, (end - start) >> PAGE_SHIFT);  in pti_set_kernel_image_nonglobal()
622 	 * We check for X86_FEATURE_PCID here. But the init-code will  in pti_init()
633 		printk(KERN_WARNING "** You are using 32-bit PTI on a 64-bit PCID-capable CPU. **\n");  in pti_init()
635 		printk(KERN_WARNING "** switch to a 64-bit kernel!                             **\n");  in pti_init()
651 	 * pti_clone_entry_text() allows those user-mode-helpers to function,  in pti_init()
660  * Finalize the kernel mappings in the userspace page-table. Some of the
664  * userspace page-table.