100 	struct bpf_prog_aux *aux;  in bpf_prog_alloc_no_stats()  local
108 	aux = kzalloc(sizeof(*aux), bpf_memcg_flags(GFP_KERNEL | gfp_extra_flags));  in bpf_prog_alloc_no_stats()
109 	if (aux == NULL) {  in bpf_prog_alloc_no_stats()
116 		kfree(aux);  in bpf_prog_alloc_no_stats()
121 	fp->aux = aux;  in bpf_prog_alloc_no_stats()
122 	fp->aux->prog = fp;  in bpf_prog_alloc_no_stats()
126 	aux->cgroup_atype = CGROUP_BPF_ATTACH_TYPE_INVALID;  in bpf_prog_alloc_no_stats()
129 	INIT_LIST_HEAD_RCU(&fp->aux->ksym.lnode);  in bpf_prog_alloc_no_stats()
131 	INIT_LIST_HEAD_RCU(&fp->aux->ksym_prefix.lnode);  in bpf_prog_alloc_no_stats()
133 	mutex_init(&fp->aux->used_maps_mutex);  in bpf_prog_alloc_no_stats()
134 	mutex_init(&fp->aux->ext_mutex);  in bpf_prog_alloc_no_stats()
135 	mutex_init(&fp->aux->dst_mutex);  in bpf_prog_alloc_no_stats()
153 		kfree(prog->aux);  in bpf_prog_alloc()
170 	if (!prog->aux->nr_linfo || !prog->jit_requested)  in bpf_prog_alloc_jited_linfo()
173 	prog->aux->jited_linfo = kvcalloc(prog->aux->nr_linfo,  in bpf_prog_alloc_jited_linfo()
174 					  sizeof(*prog->aux->jited_linfo),  in bpf_prog_alloc_jited_linfo()
176 	if (!prog->aux->jited_linfo)  in bpf_prog_alloc_jited_linfo()
184 	if (prog->aux->jited_linfo &&  in bpf_prog_jit_attempt_done()
185 	    (!prog->jited || !prog->aux->jited_linfo[0])) {  in bpf_prog_jit_attempt_done()
186 		kvfree(prog->aux->jited_linfo);  in bpf_prog_jit_attempt_done()
187 		prog->aux->jited_linfo = NULL;  in bpf_prog_jit_attempt_done()
190 	kfree(prog->aux->kfunc_tab);  in bpf_prog_jit_attempt_done()
191 	prog->aux->kfunc_tab = NULL;  in bpf_prog_jit_attempt_done()
209  *      The prog's idx to prog->aux->linfo and jited_linfo
225 	if (!prog->aux->jited_linfo || prog->aux->func_idx > prog->aux->func_cnt)  in bpf_prog_fill_jited_linfo()
229 	linfo_idx = prog->aux->linfo_idx;  in bpf_prog_fill_jited_linfo()
230 	linfo = &prog->aux->linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
234 	jited_linfo = &prog->aux->jited_linfo[linfo_idx];  in bpf_prog_fill_jited_linfo()
237 	nr_linfo = prog->aux->nr_linfo - linfo_idx;  in bpf_prog_fill_jited_linfo()
263 		fp->aux->prog = fp;  in bpf_prog_realloc()
265 		/* We keep fp->aux from fp_old around in the new  in bpf_prog_realloc()
268 		fp_old->aux = NULL;  in bpf_prog_realloc()
279 	if (fp->aux) {  in __bpf_prog_free()
280 		mutex_destroy(&fp->aux->used_maps_mutex);  in __bpf_prog_free()
281 		mutex_destroy(&fp->aux->dst_mutex);  in __bpf_prog_free()
282 		kfree(fp->aux->poke_tab);  in __bpf_prog_free()
283 		kfree(fp->aux);  in __bpf_prog_free()
464 	nr_linfo = prog->aux->nr_linfo;  in bpf_adj_linfo()
468 	linfo = prog->aux->linfo;  in bpf_adj_linfo()
560 	for (i = 0; i < fp->aux->real_func_cnt; i++)  in bpf_prog_kallsyms_del_subprogs()
561 		bpf_prog_kallsyms_del(fp->aux->func[i]);  in bpf_prog_kallsyms_del_subprogs()
583 	prog->aux->ksym.start = (unsigned long) prog->bpf_func;  in bpf_prog_ksym_set_addr()
584 	prog->aux->ksym.end   = prog->aux->ksym.start + prog->jited_len;  in bpf_prog_ksym_set_addr()
590 	char *sym = prog->aux->ksym.name;  in bpf_prog_ksym_set_name()
604 		     sizeof(prog->aux->name) > KSYM_NAME_LEN);  in bpf_prog_ksym_set_name()
609 	/* prog->aux->name will be ignored if full btf name is available */  in bpf_prog_ksym_set_name()
610 	if (prog->aux->func_info_cnt && prog->aux->func_idx < prog->aux->func_info_cnt) {  in bpf_prog_ksym_set_name()
611 		type = btf_type_by_id(prog->aux->btf,  in bpf_prog_ksym_set_name()
612 				      prog->aux->func_info[prog->aux->func_idx].type_id);  in bpf_prog_ksym_set_name()
613 		func_name = btf_name_by_offset(prog->aux->btf, type->name_off);  in bpf_prog_ksym_set_name()
618 	if (prog->aux->name[0])  in bpf_prog_ksym_set_name()
619 		snprintf(sym, (size_t)(end - sym), "_%s", prog->aux->name);  in bpf_prog_ksym_set_name()
696 	    !bpf_token_capable(fp->aux->token, CAP_BPF))  in bpf_prog_kallsyms_add()
701 	fp->aux->ksym.prog = true;  in bpf_prog_kallsyms_add()
703 	bpf_ksym_add(&fp->aux->ksym);  in bpf_prog_kallsyms_add()
713 	snprintf(fp->aux->ksym_prefix.name, KSYM_NAME_LEN,  in bpf_prog_kallsyms_add()
714 		 "__cfi_%s", fp->aux->ksym.name);  in bpf_prog_kallsyms_add()
716 	fp->aux->ksym_prefix.start = (unsigned long) fp->bpf_func - 16;  in bpf_prog_kallsyms_add()
717 	fp->aux->ksym_prefix.end   = (unsigned long) fp->bpf_func;  in bpf_prog_kallsyms_add()
719 	bpf_ksym_add(&fp->aux->ksym_prefix);  in bpf_prog_kallsyms_add()
728 	bpf_ksym_del(&fp->aux->ksym);  in bpf_prog_kallsyms_del()
732 	bpf_ksym_del(&fp->aux->ksym_prefix);  in bpf_prog_kallsyms_del()
797 	if (!prog->aux->num_exentries)  in search_bpf_extables()
800 	e = search_extable(prog->aux->extable, prog->aux->num_exentries, addr);  in search_bpf_extables()
837 	struct bpf_jit_poke_descriptor *tab = prog->aux->poke_tab;  in bpf_jit_add_poke_descriptor()
839 	u32 slot = prog->aux->size_poke_tab;  in bpf_jit_add_poke_descriptor()
862 	prog->aux->size_poke_tab = size;  in bpf_jit_add_poke_descriptor()
863 	prog->aux->poke_tab = tab;  in bpf_jit_add_poke_descriptor()
1266 		 * can pick them up from prog->aux.  in bpf_jit_get_func_addr()
1270 		else if (prog->aux->func &&  in bpf_jit_get_func_addr()
1271 			 off >= 0 && off < prog->aux->real_func_cnt)  in bpf_jit_get_func_addr()
1272 			addr = (u8 *)prog->aux->func[off]->bpf_func;  in bpf_jit_get_func_addr()
1294 			      const struct bpf_insn *aux,  in bpf_jit_blind_insn()  argument
1403 		*to++ = BPF_ALU64_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[1].imm);  in bpf_jit_blind_insn()
1406 		*to++ = BPF_ALU64_REG(BPF_MOV, aux[0].dst_reg, BPF_REG_AX);  in bpf_jit_blind_insn()
1409 		*to++ = BPF_ALU32_IMM(BPF_MOV, BPF_REG_AX, imm_rnd ^ aux[0].imm);  in bpf_jit_blind_insn()
1413 		*to++ = BPF_ALU64_REG(BPF_OR,  aux[0].dst_reg, BPF_REG_AX);  in bpf_jit_blind_insn()
1437 		/* aux->prog still points to the fp_other one, so  in bpf_prog_clone_create()
1449 	/* aux was stolen by the other clone, so we cannot free  in bpf_prog_clone_free()
1456 	fp->aux = NULL;  in bpf_prog_clone_free()
1464 	/* We have to repoint aux->prog to self, as we don't  in bpf_jit_prog_release_other()
1467 	fp->aux->prog = fp;  in bpf_jit_prog_release_other()
1473 	struct bpf_insn insn_buff[16], aux[2];  in bpf_jit_blind_constants()  local
1506 			memcpy(aux, insn, sizeof(aux));  in bpf_jit_blind_constants()
1508 		rewritten = bpf_jit_blind_insn(insn, aux, insn_buff,  in bpf_jit_blind_constants()
1509 						clone->aux->verifier_zext);  in bpf_jit_blind_constants()
1515 			/* Patching may have repointed aux->prog during  in bpf_jit_blind_constants()
2311 	struct bpf_prog_aux *aux = fp->aux;  in bpf_prog_map_compatible()  local
2321 	if (bpf_prog_is_dev_bound(aux))  in bpf_prog_map_compatible()
2331 		map->owner.xdp_has_frags = aux->xdp_has_frags;  in bpf_prog_map_compatible()
2332 		map->owner.attach_func_proto = aux->attach_func_proto;  in bpf_prog_map_compatible()
2337 		      map->owner.xdp_has_frags == aux->xdp_has_frags;  in bpf_prog_map_compatible()
2339 		    map->owner.attach_func_proto != aux->attach_func_proto) {  in bpf_prog_map_compatible()
2359 	struct bpf_prog_aux *aux = fp->aux;  in bpf_check_tail_call()  local
2362 	mutex_lock(&aux->used_maps_mutex);  in bpf_check_tail_call()
2363 	for (i = 0; i < aux->used_map_cnt; i++) {  in bpf_check_tail_call()
2364 		struct bpf_map *map = aux->used_maps[i];  in bpf_check_tail_call()
2376 	mutex_unlock(&aux->used_maps_mutex);  in bpf_check_tail_call()
2383 	u32 stack_depth = max_t(u32, fp->aux->stack_depth, 1);  in bpf_prog_select_func()
2434 	if (!bpf_prog_is_offloaded(fp->aux)) {  in bpf_prog_select_runtime()
2557 		prog_ids[i] = item->prog->aux->id;  in bpf_prog_array_copy_core()
2751 void __bpf_free_used_maps(struct bpf_prog_aux *aux,  in __bpf_free_used_maps()  argument
2758 	sleepable = aux->prog->sleepable;  in __bpf_free_used_maps()
2762 			map->ops->map_poke_untrack(map, aux);  in __bpf_free_used_maps()
2769 static void bpf_free_used_maps(struct bpf_prog_aux *aux)  in bpf_free_used_maps()  argument
2771 	__bpf_free_used_maps(aux, aux->used_maps, aux->used_map_cnt);  in bpf_free_used_maps()
2772 	kfree(aux->used_maps);  in bpf_free_used_maps()
2790 static void bpf_free_used_btfs(struct bpf_prog_aux *aux)  in bpf_free_used_btfs()  argument
2792 	__bpf_free_used_btfs(aux->used_btfs, aux->used_btf_cnt);  in bpf_free_used_btfs()
2793 	kfree(aux->used_btfs);  in bpf_free_used_btfs()
2798 	struct bpf_prog_aux *aux;  in bpf_prog_free_deferred()  local
2801 	aux = container_of(work, struct bpf_prog_aux, work);  in bpf_prog_free_deferred()
2803 	bpf_free_kfunc_btf_tab(aux->kfunc_btf_tab);  in bpf_prog_free_deferred()
2806 	if (aux->cgroup_atype != CGROUP_BPF_ATTACH_TYPE_INVALID)  in bpf_prog_free_deferred()
2807 		bpf_cgroup_atype_put(aux->cgroup_atype);  in bpf_prog_free_deferred()
2809 	bpf_free_used_maps(aux);  in bpf_prog_free_deferred()
2810 	bpf_free_used_btfs(aux);  in bpf_prog_free_deferred()
2811 	if (bpf_prog_is_dev_bound(aux))  in bpf_prog_free_deferred()
2812 		bpf_prog_dev_bound_destroy(aux->prog);  in bpf_prog_free_deferred()
2814 	if (aux->prog->has_callchain_buf)  in bpf_prog_free_deferred()
2817 	if (aux->dst_trampoline)  in bpf_prog_free_deferred()
2818 		bpf_trampoline_put(aux->dst_trampoline);  in bpf_prog_free_deferred()
2819 	for (i = 0; i < aux->real_func_cnt; i++) {  in bpf_prog_free_deferred()
2824 		aux->func[i]->aux->poke_tab = NULL;  in bpf_prog_free_deferred()
2825 		bpf_jit_free(aux->func[i]);  in bpf_prog_free_deferred()
2827 	if (aux->real_func_cnt) {  in bpf_prog_free_deferred()
2828 		kfree(aux->func);  in bpf_prog_free_deferred()
2829 		bpf_prog_unlock_free(aux->prog);  in bpf_prog_free_deferred()
2831 		bpf_jit_free(aux->prog);  in bpf_prog_free_deferred()
2837 	struct bpf_prog_aux *aux = fp->aux;  in bpf_prog_free()  local
2839 	if (aux->dst_prog)  in bpf_prog_free()
2840 		bpf_prog_put(aux->dst_prog);  in bpf_prog_free()
2841 	bpf_token_put(aux->token);  in bpf_prog_free()
2842 	INIT_WORK(&aux->work, bpf_prog_free_deferred);  in bpf_prog_free()
2843 	schedule_work(&aux->work);  in bpf_prog_free()