1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2007-2008 Steven Rostedt <[email protected]>
18 #include <linux/ftrace.h>
33 #include <asm/ftrace.h>
35 #include <asm/text-patching.h>
47 	 * ftrace has it set to "read/write".  in ftrace_arch_code_modify_prepare()
94 		return -EFAULT;  in ftrace_verify_code()
101 		return -EINVAL;  in ftrace_verify_code()
133 	unsigned long ip = rec->ip;  in ftrace_make_nop()
151 	 * x86 overrides ftrace_replace_code -- this function will never be used  in ftrace_make_nop()
155 	return -EINVAL;  in ftrace_make_nop()
160 	unsigned long ip = rec->ip;  in ftrace_make_call()
167 	return ftrace_modify_code_direct(rec->ip, old, new);  in ftrace_make_call()
182 	return -EINVAL;  in ftrace_modify_call()
222 			old = ftrace_call_replace(rec->ip, ftrace_get_addr_curr(rec));  in ftrace_replace_code()
226 		ret = ftrace_verify_code(rec->ip, old);  in ftrace_replace_code()
245 			new = ftrace_call_replace(rec->ip, ftrace_get_addr_new(rec));  in ftrace_replace_code()
253 		text_poke_queue((void *)rec->ip, new, MCOUNT_INSN_SIZE, NULL);  in ftrace_replace_code()
267 static inline void *alloc_tramp(unsigned long size)  in alloc_tramp()  argument
269 	return execmem_alloc(EXECMEM_FTRACE, size);  in alloc_tramp()
276 /* Defined as markers to the end of the ftrace default trampolines */
284 /* 0x48 0x8b 0x15 <offset-to-ftrace_trace_op (4 bytes)> */
316 	unsigned long size;  in create_trampoline()  local
326 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
340 	size = end_offset - start_offset;  in create_trampoline()
343 	 * Allocate enough size to store the ftrace_caller code,  in create_trampoline()
347 	trampoline = alloc_tramp(size + RET_SIZE + sizeof(void *));  in create_trampoline()
351 	*tramp_size = size + RET_SIZE + sizeof(void *);  in create_trampoline()
355 	ret = text_poke_copy(trampoline, (void *)start_offset, size);  in create_trampoline()
359 	ip = trampoline + size;  in create_trampoline()
366 	if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in create_trampoline()
368 		ip = trampoline + (jmp_offset - start_offset);  in create_trampoline()
383 	ptr = (unsigned long *)(trampoline + size + RET_SIZE);  in create_trampoline()
386 	op_offset -= start_offset;  in create_trampoline()
395 	offset -= (unsigned long)trampoline + op_offset + OP_REF_SIZE;  in create_trampoline()
404 	call_offset -= start_offset;  in create_trampoline()
416 	ops->flags |= FTRACE_OPS_FL_ALLOC_TRAMP;  in create_trampoline()
431 	unsigned long size;  in set_ftrace_ops_ro()  local
434 		if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in set_ftrace_ops_ro()
437 		if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {  in set_ftrace_ops_ro()
444 		size = end_offset - start_offset;  in set_ftrace_ops_ro()
445 		size = size + RET_SIZE + sizeof(void *);  in set_ftrace_ops_ro()
446 		npages = DIV_ROUND_UP(size, PAGE_SIZE);  in set_ftrace_ops_ro()
447 		set_memory_ro((unsigned long)ops->trampoline, npages);  in set_ftrace_ops_ro()
464 	return call_offset - start_offset;  in calc_trampoline_call_offset()
472 	unsigned int size;  in arch_ftrace_update_trampoline()  local
475 	if (!ops->trampoline) {  in arch_ftrace_update_trampoline()
476 		ops->trampoline = create_trampoline(ops, &size);  in arch_ftrace_update_trampoline()
477 		if (!ops->trampoline)  in arch_ftrace_update_trampoline()
479 		ops->trampoline_size = size;  in arch_ftrace_update_trampoline()
487 	if (!(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_update_trampoline()
490 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_update_trampoline()
491 	ip = ops->trampoline + offset;  in arch_ftrace_update_trampoline()
521  * If the ops->trampoline was not allocated, then it probably
522  * has a static trampoline func, or is the ftrace caller itself.
527 	bool save_regs = rec->flags & FTRACE_FL_REGS_EN;  in static_tramp_func()
530 	if (ops && ops->trampoline) {  in static_tramp_func()
537 		if (ops->trampoline == FTRACE_GRAPH_ADDR)  in static_tramp_func()
558 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_func()
561 	offset = calc_trampoline_call_offset(ops->flags & FTRACE_OPS_FL_SAVE_REGS);  in arch_ftrace_trampoline_func()
562 	return addr_from_call((void *)ops->trampoline + offset);  in arch_ftrace_trampoline_func()
567 	if (!ops || !(ops->flags & FTRACE_OPS_FL_ALLOC_TRAMP))  in arch_ftrace_trampoline_free()
570 	tramp_free((void *)ops->trampoline);  in arch_ftrace_trampoline_free()
571 	ops->trampoline = 0;  in arch_ftrace_trampoline_free()
613 	 * When resuming from suspend-to-ram, this function can be indirectly  in skip_ftrace_return()
627 	if (atomic_read(&current->tracing_graph_pause))  in skip_ftrace_return()
652 	struct pt_regs *regs = &arch_ftrace_regs(fregs)->regs;  in ftrace_graph_func()