1 // SPDX-License-Identifier: GPL-2.0-only
5  * This contains most of the x86 vDSO kernel-side code.
44 int __init init_vdso_image(const struct vdso_image *image)  in init_vdso_image()  argument
47 	BUG_ON(image->size % PAGE_SIZE != 0);  in init_vdso_image()
49 	apply_alternatives((struct alt_instr *)(image->data + image->alt),  in init_vdso_image()
50 			   (struct alt_instr *)(image->data + image->alt +  in init_vdso_image()
51 						image->alt_len),  in init_vdso_image()
63 	const struct vdso_image *image = vma->vm_mm->context.vdso_image;  in vdso_fault()  local
65 	if (!image || (vmf->pgoff << PAGE_SHIFT) >= image->size)  in vdso_fault()
68 	vmf->page = virt_to_page(image->data + (vmf->pgoff << PAGE_SHIFT));  in vdso_fault()
69 	get_page(vmf->page);  in vdso_fault()
73 static void vdso_fix_landing(const struct vdso_image *image,  in vdso_fix_landing()  argument
77 	if (in_ia32_syscall() && image == &vdso_image_32) {  in vdso_fix_landing()
79 		unsigned long vdso_land = image->sym_int80_landing_pad;  in vdso_fix_landing()
81 			(unsigned long)current->mm->context.vdso;  in vdso_fix_landing()
83 		/* Fixing userspace landing - look at do_fast_syscall_32 */  in vdso_fix_landing()
84 		if (regs->ip == old_land_addr)  in vdso_fix_landing()
85 			regs->ip = new_vma->vm_start + vdso_land;  in vdso_fix_landing()
93 	const struct vdso_image *image = current->mm->context.vdso_image;  in vdso_mremap()  local
95 	vdso_fix_landing(image, new_vma);  in vdso_mremap()
96 	current->mm->context.vdso = (void __user *)new_vma->vm_start;  in vdso_mremap()
104  * non-root time namespace. Whenever a task changes its namespace, the VVAR
105  * page tables are cleared and then they will re-faulted with a
111 	struct mm_struct *mm = task->mm;  in vdso_join_timens()
129 	const struct vdso_image *image = vma->vm_mm->context.vdso_image;  in vvar_fault()  local
133 	if (!image)  in vvar_fault()
136 	sym_offset = (long)(vmf->pgoff << PAGE_SHIFT) +  in vvar_fault()
137 		image->sym_vvar_start;  in vvar_fault()
141 	 * does not exist for this vdso image, not that the page is at  in vvar_fault()
149 	if (sym_offset == image->sym_vvar_page) {  in vvar_fault()
156 		 * specific VVAR is mapped with the sym_vvar_page offset and  in vvar_fault()
166 			 * Optimization: inside time namespace pre-fault  in vvar_fault()
171 			addr = vmf->address + (image->sym_timens_page - sym_offset);  in vvar_fault()
179 		return vmf_insert_pfn(vma, vmf->address, pfn);  in vvar_fault()
181 	} else if (sym_offset == image->sym_timens_page) {  in vvar_fault()
188 		return vmf_insert_pfn(vma, vmf->address, pfn);  in vvar_fault()
197 	switch (vmf->pgoff) {  in vvar_vclock_fault()
205 			return vmf_insert_pfn_prot(vma, vmf->address,  in vvar_vclock_fault()
207 					pgprot_decrypted(vma->vm_page_prot));  in vvar_vclock_fault()
217 			return vmf_insert_pfn(vma, vmf->address, pfn);  in vvar_vclock_fault()
242  * @image          - blob to map
243  * @addr           - request a specific address (zero to map at free addr)
245 static int map_vdso(const struct vdso_image *image, unsigned long addr)  in map_vdso()  argument
247 	struct mm_struct *mm = current->mm;  in map_vdso()
253 		return -EINTR;  in map_vdso()
256 				 image->size - image->sym_vvar_start, 0, 0);  in map_vdso()
262 	text_start = addr - image->sym_vvar_start;  in map_vdso()
269 				       image->size,  in map_vdso()
281 				       (__VVAR_PAGES - VDSO_NR_VCLOCK_PAGES) * PAGE_SIZE,  in map_vdso()
288 		do_munmap(mm, text_start, image->size, NULL);  in map_vdso()
301 		do_munmap(mm, text_start, image->size, NULL);  in map_vdso()
302 		do_munmap(mm, addr, image->size, NULL);  in map_vdso()
306 	current->mm->context.vdso = (void __user *)text_start;  in map_vdso()
307 	current->mm->context.vdso_image = image;  in map_vdso()
314 int map_vdso_once(const struct vdso_image *image, unsigned long addr)  in map_vdso_once()  argument
316 	struct mm_struct *mm = current->mm;  in map_vdso_once()
322 	 * Check if we have already mapped vdso blob - fail to prevent  in map_vdso_once()
333 			return -EEXIST;  in map_vdso_once()
338 	return map_vdso(image, addr);  in map_vdso_once()
388 	const struct vdso_image *image = current->mm->context.vdso_image;  in arch_syscall_is_vdso_sigreturn()  local
389 	unsigned long vdso = (unsigned long) current->mm->context.vdso;  in arch_syscall_is_vdso_sigreturn()
391 	if (in_ia32_syscall() && image == &vdso_image_32) {  in arch_syscall_is_vdso_sigreturn()
392 		if (regs->ip == vdso + image->sym_vdso32_sigreturn_landing_pad ||  in arch_syscall_is_vdso_sigreturn()
393 		    regs->ip == vdso + image->sym_vdso32_rt_sigreturn_landing_pad)  in arch_syscall_is_vdso_sigreturn()