1 // SPDX-License-Identifier: GPL-2.0
3  * fprobe - Simple ftrace probe wrapper for function entry.
26 #define SIZE_IN_LONG(x) ((x + sizeof(long) - 1) >> (sizeof(long) == 8 ? 3 : 2))
32  *   instance related to the funciton address. The key is the ftrace IP
40  *  - Normal hlist traversal and RCU add/del under 'fprobe_mutex' is held.
41  *  - RCU hlist traversal under disabling preempt
49  * probe, once for correcting the total required size, the second time is
62 		if (node->addr == ip)  in find_first_fprobe_node()
72 	unsigned long ip = node->addr;  in insert_fprobe_node()
80 		hlist_add_before_rcu(&node->hlist, &next->hlist);  in insert_fprobe_node()
84 	hlist_add_head_rcu(&node->hlist, head);  in insert_fprobe_node()
93 	if (READ_ONCE(node->fp) != NULL) {  in delete_fprobe_node()
94 		WRITE_ONCE(node->fp, NULL);  in delete_fprobe_node()
95 		hlist_del_rcu(&node->hlist);  in delete_fprobe_node()
97 	return !!find_first_fprobe_node(node->addr);  in delete_fprobe_node()
109 		if (fph->fp == fp)  in is_fprobe_still_exist()
118 	struct fprobe_hlist *fph = fp->hlist_array;  in add_fprobe_hash()
124 		return -EINVAL;  in add_fprobe_hash()
127 		return -EEXIST;  in add_fprobe_hash()
130 	hlist_add_head_rcu(&fp->hlist_array->hlist, head);  in add_fprobe_hash()
136 	struct fprobe_hlist *fph = fp->hlist_array;  in del_fprobe_hash()
141 		return -EINVAL;  in del_fprobe_hash()
144 		return -ENOENT;  in del_fprobe_hash()
146 	fph->fp = NULL;  in del_fprobe_hash()
147 	hlist_del_rcu(&fph->hlist);  in del_fprobe_hash()
192 	fph->fp = fp;  in write_fprobe_header()
193 	fph->size_words = size_words;  in write_fprobe_header()
202 	*fp = fph->fp;  in read_fprobe_header()
203 	*size_words = fph->size_words;  in read_fprobe_header()
214  * the shadow stack with its entry data size.
221 	if (!fp->entry_handler)  in __fprobe_handler()
224 	return fp->entry_handler(fp, ip, parent_ip, fregs, data);  in __fprobe_handler()
239 		fp->nmissed++;  in __fprobe_kprobe_handler()
254 	unsigned long func = trace->func;  in fprobe_entry()
269 		if (node->addr != func)  in fprobe_entry()
271 		fp = READ_ONCE(node->fp);  in fprobe_entry()
272 		if (!fp || !fp->exit_handler)  in fprobe_entry()
279 			FPROBE_HEADER_SIZE_IN_LONG + SIZE_IN_LONG(fp->entry_data_size);  in fprobe_entry()
283 		fgraph_data = fgraph_reserve_data(gops->idx, reserved_words * sizeof(long));  in fprobe_entry()
286 				if (node->addr != func)  in fprobe_entry()
288 				fp = READ_ONCE(node->fp);  in fprobe_entry()
290 					fp->nmissed++;  in fprobe_entry()
306 		if (node->addr != func)  in fprobe_entry()
308 		fp = READ_ONCE(node->fp);  in fprobe_entry()
312 		data_size = fp->entry_data_size;  in fprobe_entry()
313 		if (data_size && fp->exit_handler)  in fprobe_entry()
324 		if (!ret && fp->exit_handler) {  in fprobe_entry()
332 		memset(fgraph_data + used, 0, reserved_words - used);  in fprobe_entry()
346 	int size, curr;  in fprobe_return()  local
349 	fgraph_data = (unsigned long *)fgraph_retrieve_data(gops->idx, &size);  in fprobe_return()
352 	size_words = SIZE_IN_LONG(size);  in fprobe_return()
359 		read_fprobe_header(&fgraph_data[curr], &fp, &size);  in fprobe_return()
364 			if (WARN_ON_ONCE(curr + size > size_words))  in fprobe_return()
366 			fp->exit_handler(fp, trace->func, ret_ip, fregs,  in fprobe_return()
367 					 size ? fgraph_data + curr : NULL);  in fprobe_return()
369 		curr += size;  in fprobe_return()
381 /* Add @addrs to the ftrace filter and register fgraph if needed. */
403 /* Remove @addrs from the ftrace filter and unregister fgraph if possible. */
408 	fprobe_graph_active--;  in fprobe_graph_remove_ips()
423 	int size;  member
431 	if (alist->index >= alist->size)  in fprobe_addr_list_add()
432 		return -ENOMEM;  in fprobe_addr_list_add()
434 	alist->addrs[alist->index++] = addr;  in fprobe_addr_list_add()
435 	if (alist->index < alist->size)  in fprobe_addr_list_add()
439 	addrs = kcalloc(alist->size * 2, sizeof(*addrs), GFP_KERNEL);  in fprobe_addr_list_add()
441 		return -ENOMEM;  in fprobe_addr_list_add()
443 	memcpy(addrs, alist->addrs, alist->size * sizeof(*addrs));  in fprobe_addr_list_add()
444 	alist->size *= 2;  in fprobe_addr_list_add()
445 	kfree(alist->addrs);  in fprobe_addr_list_add()
446 	alist->addrs = addrs;  in fprobe_addr_list_add()
458 		if (!within_module(node->addr, mod))  in fprobe_remove_node_in_module()
467 			ret = fprobe_addr_list_add(alist, node->addr);  in fprobe_remove_node_in_module()
475 	struct fprobe_addr_list alist = {.size = FPROBE_IPS_BATCH_INIT};  in fprobe_module_callback()
482 	alist.addrs = kcalloc(alist.size, sizeof(*alist.addrs), GFP_KERNEL);  in fprobe_module_callback()
491 	if (alist.index < alist.size && alist.index > 0)  in fprobe_module_callback()
521 /* Convert ftrace location address from symbols */
529 		return ERR_PTR(-ENOMEM);  in get_ftrace_locations()
538 	return ERR_PTR(-ENOENT);  in get_ftrace_locations()
545 	size_t size;  member
554 	if (!glob_match(match->filter, name) ||  in filter_match_callback()
555 	    (match->notfilter && glob_match(match->notfilter, name)))  in filter_match_callback()
561 	if (match->addrs) {  in filter_match_callback()
567 		match->mods[match->index] = mod;  in filter_match_callback()
568 		match->addrs[match->index] = addr;  in filter_match_callback()
570 	match->index++;  in filter_match_callback()
571 	return match->index == match->size;  in filter_match_callback()
575  * Make IP list from the filter/no-filter glob patterns.
578  * is passed with an array, we need to pass the an @mods array of the same size
585 			       size_t size)  in get_ips_from_filter()  argument
588 		.index = 0, .size = size, .addrs = addrs, .mods = mods};  in get_ips_from_filter()
592 		return -EINVAL;  in get_ips_from_filter()
603 	return match.index ?: -ENOENT;  in get_ips_from_filter()
608 	kfree(fp->hlist_array);  in fprobe_fail_cleanup()
609 	fp->hlist_array = NULL;  in fprobe_fail_cleanup()
617 	int size, i;  in fprobe_init()  local
620 		return -EINVAL;  in fprobe_init()
622 	size = ALIGN(fp->entry_data_size, sizeof(long));  in fprobe_init()
623 	if (size > MAX_FPROBE_DATA_SIZE)  in fprobe_init()
624 		return -E2BIG;  in fprobe_init()
625 	fp->entry_data_size = size;  in fprobe_init()
629 		return -ENOMEM;  in fprobe_init()
631 	fp->nmissed = 0;  in fprobe_init()
633 	hlist_array->size = num;  in fprobe_init()
634 	fp->hlist_array = hlist_array;  in fprobe_init()
635 	hlist_array->fp = fp;  in fprobe_init()
637 		hlist_array->array[i].fp = fp;  in fprobe_init()
641 			return -ENOENT;  in fprobe_init()
643 		hlist_array->array[i].addr = addr;  in fprobe_init()
651  * register_fprobe() - Register fprobe to ftrace by pattern.
656  * Register @fp to ftrace for enabling the probe on the symbols matched to @filter.
659  * Return 0 if @fp is registered successfully, -errno if not.
668 		return -EINVAL;  in register_fprobe()
676 		return -ENOMEM;  in register_fprobe()
680 		return -ENOMEM;  in register_fprobe()
697  * register_fprobe_ips() - Register fprobe to ftrace by address.
702  * Register @fp to ftrace for enabling the probe on the address given by @addrs.
703  * The @addrs must be the addresses of ftrace location address, which may be
704  * the symbol address + arch-dependent offset.
707  * Return 0 if @fp is registered successfully, -errno if not.
720 	hlist_array = fp->hlist_array;  in register_fprobe_ips()
724 		for (i = 0; i < hlist_array->size; i++)  in register_fprobe_ips()
725 			insert_fprobe_node(&hlist_array->array[i]);  in register_fprobe_ips()
737  * register_fprobe_syms() - Register fprobe to ftrace by symbols.
745  * Return 0 if @fp is registered successfully, -errno if not.
753 		return -EINVAL;  in register_fprobe_syms()
769 	if (!fp || !fp->hlist_array)  in fprobe_is_registered()
775  * unregister_fprobe() - Unregister fprobe.
778  * Unregister fprobe (and remove ftrace hooks from the function entries).
780  * Return 0 if @fp is unregistered successfully, -errno if not.
790 		ret = -EINVAL;  in unregister_fprobe()
794 	hlist_array = fp->hlist_array;  in unregister_fprobe()
795 	addrs = kcalloc(hlist_array->size, sizeof(unsigned long), GFP_KERNEL);  in unregister_fprobe()
797 		ret = -ENOMEM;	/* TODO: Fallback to one-by-one loop */  in unregister_fprobe()
801 	/* Remove non-synonim ips from table and hash */  in unregister_fprobe()
803 	for (i = 0; i < hlist_array->size; i++) {  in unregister_fprobe()
804 		if (!delete_fprobe_node(&hlist_array->array[i]))  in unregister_fprobe()
805 			addrs[count++] = hlist_array->array[i].addr;  in unregister_fprobe()
812 	fp->hlist_array = NULL;  in unregister_fprobe()