1 // SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause)
6  * Copyright (C) 2013-2015 Alexei Starovoitov <[email protected]>
70 #pragma GCC diagnostic ignored "-Wformat-nonliteral"
220 	[BPF_PROG_TYPE_EXT]			= "ext",
272 	if (err != -EPERM || geteuid() != 0)  in pr_perm_msg()
289 	pr_warn("permission error while running as root; try raising 'ulimit -l'? current value: %s\n",  in pr_perm_msg()
305 	fd = -1;			\
316 	/* as of v1.0 libbpf_set_strict_mode() is a no-op */  in libbpf_set_strict_mode()
362 /* stored as sec_def->cookie for all libbpf-supported SEC()s */
381 	/* BPF program support non-linear XDP buffer */
419 	 * program. For the entry-point (main) BPF program, this is always
420 	 * zero. For a sub-program, this gets reset before each of main BPF
422 	 * whether sub-program was already appended to the main program, and
434 	 * entry-point BPF programs this includes the size of main program
435 	 * itself plus all the used sub-programs, appended at the end
486 	 * kern_vdata-size == sizeof(struct bpf_struct_ops_tcp_congestion_ops)
497 #define KCONFIG_SEC ".kconfig"
523 	 * .data*, .bss, .kconfig) and preserves their original ELF section
594 			/* BTF fd index to be patched in for insn->off, this is
595 			 * 0 for vmlinux BTF, index in obj->fd_array for module
659 	char *kconfig;  member
680 	/* Path to the custom BTF to be used for BPF CO-RE relocations as an
684 	/* vmlinux BTF override for CO-RE relocations */
729 	zclose(prog->fd);  in bpf_program__unload()
731 	zfree(&prog->func_info);  in bpf_program__unload()
732 	zfree(&prog->line_info);  in bpf_program__unload()
741 	zfree(&prog->name);  in bpf_program__exit()
742 	zfree(&prog->sec_name);  in bpf_program__exit()
743 	zfree(&prog->insns);  in bpf_program__exit()
744 	zfree(&prog->reloc_desc);  in bpf_program__exit()
746 	prog->nr_reloc = 0;  in bpf_program__exit()
747 	prog->insns_cnt = 0;  in bpf_program__exit()
748 	prog->sec_idx = -1;  in bpf_program__exit()
753 	return BPF_CLASS(insn->code) == BPF_JMP &&  in insn_is_subprog_call()
754 	       BPF_OP(insn->code) == BPF_CALL &&  in insn_is_subprog_call()
755 	       BPF_SRC(insn->code) == BPF_K &&  in insn_is_subprog_call()
756 	       insn->src_reg == BPF_PSEUDO_CALL &&  in insn_is_subprog_call()
757 	       insn->dst_reg == 0 &&  in insn_is_subprog_call()
758 	       insn->off == 0;  in insn_is_subprog_call()
763 	return insn->code == (BPF_JMP | BPF_CALL);  in is_call_insn()
768 	return is_ldimm64_insn(insn) && insn->src_reg == BPF_PSEUDO_FUNC;  in insn_is_pseudo_func()
779 		return -EINVAL;  in bpf_object__init_prog()
783 	prog->obj = obj;  in bpf_object__init_prog()
785 	prog->sec_idx = sec_idx;  in bpf_object__init_prog()
786 	prog->sec_insn_off = sec_off / BPF_INSN_SZ;  in bpf_object__init_prog()
787 	prog->sec_insn_cnt = insn_data_sz / BPF_INSN_SZ;  in bpf_object__init_prog()
789 	prog->insns_cnt = prog->sec_insn_cnt;  in bpf_object__init_prog()
791 	prog->type = BPF_PROG_TYPE_UNSPEC;  in bpf_object__init_prog()
792 	prog->fd = -1;  in bpf_object__init_prog()
793 	prog->exception_cb_idx = -1;  in bpf_object__init_prog()
800 		prog->autoload = false;  in bpf_object__init_prog()
804 		prog->autoload = true;  in bpf_object__init_prog()
807 	prog->autoattach = true;  in bpf_object__init_prog()
810 	prog->log_level = obj->log_level;  in bpf_object__init_prog()
812 	prog->sec_name = strdup(sec_name);  in bpf_object__init_prog()
813 	if (!prog->sec_name)  in bpf_object__init_prog()
816 	prog->name = strdup(name);  in bpf_object__init_prog()
817 	if (!prog->name)  in bpf_object__init_prog()
820 	prog->insns = malloc(insn_data_sz);  in bpf_object__init_prog()
821 	if (!prog->insns)  in bpf_object__init_prog()
823 	memcpy(prog->insns, insn_data, insn_data_sz);  in bpf_object__init_prog()
829 	return -ENOMEM;  in bpf_object__init_prog()
836 	Elf_Data *symbols = obj->efile.symbols;  in bpf_object__add_programs()
838 	void *data = sec_data->d_buf;  in bpf_object__add_programs()
839 	size_t sec_sz = sec_data->d_size, sec_off, prog_sz, nr_syms;  in bpf_object__add_programs()
844 	progs = obj->programs;  in bpf_object__add_programs()
845 	nr_progs = obj->nr_programs;  in bpf_object__add_programs()
846 	nr_syms = symbols->d_size / sizeof(Elf64_Sym);  in bpf_object__add_programs()
851 		if (sym->st_shndx != sec_idx)  in bpf_object__add_programs()
853 		if (ELF64_ST_TYPE(sym->st_info) != STT_FUNC)  in bpf_object__add_programs()
856 		prog_sz = sym->st_size;  in bpf_object__add_programs()
857 		sec_off = sym->st_value;  in bpf_object__add_programs()
859 		name = elf_sym_str(obj, sym->st_name);  in bpf_object__add_programs()
863 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__add_programs()
869 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__add_programs()
872 		if (sec_idx != obj->efile.text_shndx && ELF64_ST_BIND(sym->st_info) == STB_LOCAL) {  in bpf_object__add_programs()
874 			return -ENOTSUP;  in bpf_object__add_programs()
883 			 * In this case the original obj->programs  in bpf_object__add_programs()
889 			return -ENOMEM;  in bpf_object__add_programs()
891 		obj->programs = progs;  in bpf_object__add_programs()
900 		if (ELF64_ST_BIND(sym->st_info) != STB_LOCAL)  in bpf_object__add_programs()
901 			prog->sym_global = true;  in bpf_object__add_programs()
908 		if (prog->sym_global && (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN  in bpf_object__add_programs()
909 		    || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL))  in bpf_object__add_programs()
910 			prog->mark_btf_static = true;  in bpf_object__add_programs()
913 		obj->nr_programs = nr_progs;  in bpf_object__add_programs()
941 		if (!strcmp(btf__name_by_offset(btf, m->name_off), name))  in find_member_by_name()
1004 		if (kern_data_member->type == kern_type_id)  in find_struct_ops_kern_types()
1010 		return -EINVAL;  in find_struct_ops_kern_types()
1024 	return map->def.type == BPF_MAP_TYPE_STRUCT_OPS;  in bpf_map__is_struct_ops()
1032 	for (i = 0; i < obj->nr_programs; i++) {  in is_valid_st_ops_program()
1033 		if (&obj->programs[i] == prog)  in is_valid_st_ops_program()
1034 			return prog->type == BPF_PROG_TYPE_STRUCT_OPS;  in is_valid_st_ops_program()
1051 	for (i = 0; i < obj->nr_programs; ++i) {  in bpf_object_adjust_struct_ops_autoload()
1055 		prog = &obj->programs[i];  in bpf_object_adjust_struct_ops_autoload()
1056 		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS)  in bpf_object_adjust_struct_ops_autoload()
1059 		for (j = 0; j < obj->nr_maps; ++j) {  in bpf_object_adjust_struct_ops_autoload()
1060 			map = &obj->maps[j];  in bpf_object_adjust_struct_ops_autoload()
1064 			vlen = btf_vlen(map->st_ops->type);  in bpf_object_adjust_struct_ops_autoload()
1066 				slot_prog = map->st_ops->progs[k];  in bpf_object_adjust_struct_ops_autoload()
1071 				if (map->autocreate)  in bpf_object_adjust_struct_ops_autoload()
1076 			prog->autoload = should_load;  in bpf_object_adjust_struct_ops_autoload()
1088 	struct bpf_object *obj = map->obj;  in bpf_map__init_kern_struct_ops()
1089 	const struct btf *btf = obj->btf;  in bpf_map__init_kern_struct_ops()
1097 	st_ops = map->st_ops;  in bpf_map__init_kern_struct_ops()
1098 	type = st_ops->type;  in bpf_map__init_kern_struct_ops()
1099 	tname = st_ops->tname;  in bpf_map__init_kern_struct_ops()
1107 	kern_btf = mod_btf ? mod_btf->btf : obj->btf_vmlinux;  in bpf_map__init_kern_struct_ops()
1110 		 map->name, st_ops->type_id, kern_type_id, kern_vtype_id);  in bpf_map__init_kern_struct_ops()
1112 	map->mod_btf_fd = mod_btf ? mod_btf->fd : -1;  in bpf_map__init_kern_struct_ops()
1113 	map->def.value_size = kern_vtype->size;  in bpf_map__init_kern_struct_ops()
1114 	map->btf_vmlinux_value_type_id = kern_vtype_id;  in bpf_map__init_kern_struct_ops()
1116 	st_ops->kern_vdata = calloc(1, kern_vtype->size);  in bpf_map__init_kern_struct_ops()
1117 	if (!st_ops->kern_vdata)  in bpf_map__init_kern_struct_ops()
1118 		return -ENOMEM;  in bpf_map__init_kern_struct_ops()
1120 	data = st_ops->data;  in bpf_map__init_kern_struct_ops()
1121 	kern_data_off = kern_data_member->offset / 8;  in bpf_map__init_kern_struct_ops()
1122 	kern_data = st_ops->kern_vdata + kern_data_off;  in bpf_map__init_kern_struct_ops()
1135 		mname = btf__name_by_offset(btf, member->name_off);  in bpf_map__init_kern_struct_ops()
1136 		moff = member->offset / 8;  in bpf_map__init_kern_struct_ops()
1138 		msize = btf__resolve_size(btf, member->type);  in bpf_map__init_kern_struct_ops()
1141 				map->name, mname);  in bpf_map__init_kern_struct_ops()
1149 					map->name, mname);  in bpf_map__init_kern_struct_ops()
1150 				return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1153 			if (st_ops->progs[i]) {  in bpf_map__init_kern_struct_ops()
1161 				st_ops->progs[i]->autoload = false;  in bpf_map__init_kern_struct_ops()
1162 				st_ops->progs[i] = NULL;  in bpf_map__init_kern_struct_ops()
1165 			/* Skip all-zero/NULL fields if they are not present in the kernel BTF */  in bpf_map__init_kern_struct_ops()
1167 				map->name, mname);  in bpf_map__init_kern_struct_ops()
1171 		kern_member_idx = kern_member - btf_members(kern_type);  in bpf_map__init_kern_struct_ops()
1175 				map->name, mname);  in bpf_map__init_kern_struct_ops()
1176 			return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1179 		kern_moff = kern_member->offset / 8;  in bpf_map__init_kern_struct_ops()
1182 		mtype = skip_mods_and_typedefs(btf, member->type, &mtype_id);  in bpf_map__init_kern_struct_ops()
1183 		kern_mtype = skip_mods_and_typedefs(kern_btf, kern_member->type,  in bpf_map__init_kern_struct_ops()
1185 		if (BTF_INFO_KIND(mtype->info) !=  in bpf_map__init_kern_struct_ops()
1186 		    BTF_INFO_KIND(kern_mtype->info)) {  in bpf_map__init_kern_struct_ops()
1188 				map->name, mname, BTF_INFO_KIND(mtype->info),  in bpf_map__init_kern_struct_ops()
1189 				BTF_INFO_KIND(kern_mtype->info));  in bpf_map__init_kern_struct_ops()
1190 			return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1199 			if (st_ops->progs[i] && st_ops->progs[i] != prog)  in bpf_map__init_kern_struct_ops()
1200 				st_ops->progs[i]->autoload = false;  in bpf_map__init_kern_struct_ops()
1203 			st_ops->progs[i] = prog;  in bpf_map__init_kern_struct_ops()
1209 					map->name, mname);  in bpf_map__init_kern_struct_ops()
1210 				return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1214 							    kern_mtype->type,  in bpf_map__init_kern_struct_ops()
1217 			/* mtype->type must be a func_proto which was  in bpf_map__init_kern_struct_ops()
1223 					map->name, mname);  in bpf_map__init_kern_struct_ops()
1224 				return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1228 				prog->attach_btf_obj_fd = mod_btf->fd;  in bpf_map__init_kern_struct_ops()
1233 			if (!prog->attach_btf_id) {  in bpf_map__init_kern_struct_ops()
1234 				prog->attach_btf_id = kern_type_id;  in bpf_map__init_kern_struct_ops()
1235 				prog->expected_attach_type = kern_member_idx;  in bpf_map__init_kern_struct_ops()
1238 			/* struct_ops BPF prog can be re-used between multiple  in bpf_map__init_kern_struct_ops()
1243 			if (prog->attach_btf_id != kern_type_id) {  in bpf_map__init_kern_struct_ops()
1245 					map->name, mname, prog->name, prog->sec_name, prog->type,  in bpf_map__init_kern_struct_ops()
1246 					prog->attach_btf_id, kern_type_id);  in bpf_map__init_kern_struct_ops()
1247 				return -EINVAL;  in bpf_map__init_kern_struct_ops()
1249 			if (prog->expected_attach_type != kern_member_idx) {  in bpf_map__init_kern_struct_ops()
1251 					map->name, mname, prog->name, prog->sec_name, prog->type,  in bpf_map__init_kern_struct_ops()
1252 					prog->expected_attach_type, kern_member_idx);  in bpf_map__init_kern_struct_ops()
1253 				return -EINVAL;  in bpf_map__init_kern_struct_ops()
1256 			st_ops->kern_func_off[i] = kern_data_off + kern_moff;  in bpf_map__init_kern_struct_ops()
1259 				 map->name, mname, prog->name, moff,  in bpf_map__init_kern_struct_ops()
1268 				map->name, mname, (ssize_t)msize,  in bpf_map__init_kern_struct_ops()
1270 			return -ENOTSUP;  in bpf_map__init_kern_struct_ops()
1274 			 map->name, mname, (unsigned int)msize,  in bpf_map__init_kern_struct_ops()
1288 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object__init_kern_struct_ops_maps()
1289 		map = &obj->maps[i];  in bpf_object__init_kern_struct_ops_maps()
1294 		if (!map->autocreate)  in bpf_object__init_kern_struct_ops_maps()
1317 	if (shndx == -1)  in init_struct_ops_maps()
1320 	btf = obj->btf;  in init_struct_ops_maps()
1326 		return -EINVAL;  in init_struct_ops_maps()
1332 		type = btf__type_by_id(obj->btf, vsi->type);  in init_struct_ops_maps()
1333 		var_name = btf__name_by_offset(obj->btf, type->name_off);  in init_struct_ops_maps()
1335 		type_id = btf__resolve_type(obj->btf, vsi->type);  in init_struct_ops_maps()
1338 				vsi->type, sec_name);  in init_struct_ops_maps()
1339 			return -EINVAL;  in init_struct_ops_maps()
1342 		type = btf__type_by_id(obj->btf, type_id);  in init_struct_ops_maps()
1343 		tname = btf__name_by_offset(obj->btf, type->name_off);  in init_struct_ops_maps()
1346 			return -ENOTSUP;  in init_struct_ops_maps()
1350 			return -EINVAL;  in init_struct_ops_maps()
1357 		map->sec_idx = shndx;  in init_struct_ops_maps()
1358 		map->sec_offset = vsi->offset;  in init_struct_ops_maps()
1359 		map->name = strdup(var_name);  in init_struct_ops_maps()
1360 		if (!map->name)  in init_struct_ops_maps()
1361 			return -ENOMEM;  in init_struct_ops_maps()
1362 		map->btf_value_type_id = type_id;  in init_struct_ops_maps()
1368 			map->autocreate = false;  in init_struct_ops_maps()
1373 		map->def.type = BPF_MAP_TYPE_STRUCT_OPS;  in init_struct_ops_maps()
1374 		map->def.key_size = sizeof(int);  in init_struct_ops_maps()
1375 		map->def.value_size = type->size;  in init_struct_ops_maps()
1376 		map->def.max_entries = 1;  in init_struct_ops_maps()
1377 		map->def.map_flags = strcmp(sec_name, STRUCT_OPS_LINK_SEC) == 0 ? BPF_F_LINK : 0;  in init_struct_ops_maps()
1379 		map->st_ops = calloc(1, sizeof(*map->st_ops));  in init_struct_ops_maps()
1380 		if (!map->st_ops)  in init_struct_ops_maps()
1381 			return -ENOMEM;  in init_struct_ops_maps()
1382 		st_ops = map->st_ops;  in init_struct_ops_maps()
1383 		st_ops->data = malloc(type->size);  in init_struct_ops_maps()
1384 		st_ops->progs = calloc(btf_vlen(type), sizeof(*st_ops->progs));  in init_struct_ops_maps()
1385 		st_ops->kern_func_off = malloc(btf_vlen(type) *  in init_struct_ops_maps()
1386 					       sizeof(*st_ops->kern_func_off));  in init_struct_ops_maps()
1387 		if (!st_ops->data || !st_ops->progs || !st_ops->kern_func_off)  in init_struct_ops_maps()
1388 			return -ENOMEM;  in init_struct_ops_maps()
1390 		if (vsi->offset + type->size > data->d_size) {  in init_struct_ops_maps()
1393 			return -EINVAL;  in init_struct_ops_maps()
1396 		memcpy(st_ops->data,  in init_struct_ops_maps()
1397 		       data->d_buf + vsi->offset,  in init_struct_ops_maps()
1398 		       type->size);  in init_struct_ops_maps()
1399 		st_ops->tname = tname;  in init_struct_ops_maps()
1400 		st_ops->type = type;  in init_struct_ops_maps()
1401 		st_ops->type_id = type_id;  in init_struct_ops_maps()
1404 			 tname, type_id, var_name, vsi->offset);  in init_struct_ops_maps()
1415 	for (sec_idx = 0; sec_idx < obj->efile.sec_cnt; ++sec_idx) {  in bpf_object_init_struct_ops()
1416 		struct elf_sec_desc *desc = &obj->efile.secs[sec_idx];  in bpf_object_init_struct_ops()
1418 		if (desc->sec_type != SEC_ST_OPS)  in bpf_object_init_struct_ops()
1423 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object_init_struct_ops()
1425 		err = init_struct_ops_maps(obj, sec_name, sec_idx, desc->data);  in bpf_object_init_struct_ops()
1444 		return ERR_PTR(-ENOMEM);  in bpf_object__new()
1447 	strcpy(obj->path, path);  in bpf_object__new()
1449 		libbpf_strlcpy(obj->name, obj_name, sizeof(obj->name));  in bpf_object__new()
1452 		libbpf_strlcpy(obj->name, basename((void *)path), sizeof(obj->name));  in bpf_object__new()
1453 		end = strchr(obj->name, '.');  in bpf_object__new()
1458 	obj->efile.fd = -1;  in bpf_object__new()
1465 	obj->efile.obj_buf = obj_buf;  in bpf_object__new()
1466 	obj->efile.obj_buf_sz = obj_buf_sz;  in bpf_object__new()
1467 	obj->efile.btf_maps_shndx = -1;  in bpf_object__new()
1468 	obj->kconfig_map_idx = -1;  in bpf_object__new()
1470 	obj->kern_version = get_kernel_version();  in bpf_object__new()
1471 	obj->loaded = false;  in bpf_object__new()
1478 	if (!obj->efile.elf)  in bpf_object__elf_finish()
1481 	elf_end(obj->efile.elf);  in bpf_object__elf_finish()
1482 	obj->efile.elf = NULL;  in bpf_object__elf_finish()
1483 	obj->efile.symbols = NULL;  in bpf_object__elf_finish()
1484 	obj->efile.arena_data = NULL;  in bpf_object__elf_finish()
1486 	zfree(&obj->efile.secs);  in bpf_object__elf_finish()
1487 	obj->efile.sec_cnt = 0;  in bpf_object__elf_finish()
1488 	zclose(obj->efile.fd);  in bpf_object__elf_finish()
1489 	obj->efile.obj_buf = NULL;  in bpf_object__elf_finish()
1490 	obj->efile.obj_buf_sz = 0;  in bpf_object__elf_finish()
1499 	if (obj->efile.elf) {  in bpf_object__elf_init()
1501 		return -LIBBPF_ERRNO__LIBELF;  in bpf_object__elf_init()
1504 	if (obj->efile.obj_buf_sz > 0) {  in bpf_object__elf_init()
1506 		elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz);  in bpf_object__elf_init()
1508 		obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC);  in bpf_object__elf_init()
1509 		if (obj->efile.fd < 0) {  in bpf_object__elf_init()
1512 			err = -errno;  in bpf_object__elf_init()
1514 			pr_warn("elf: failed to open %s: %s\n", obj->path, cp);  in bpf_object__elf_init()
1518 		elf = elf_begin(obj->efile.fd, ELF_C_READ_MMAP, NULL);  in bpf_object__elf_init()
1522 		pr_warn("elf: failed to open %s as ELF file: %s\n", obj->path, elf_errmsg(-1));  in bpf_object__elf_init()
1523 		err = -LIBBPF_ERRNO__LIBELF;  in bpf_object__elf_init()
1527 	obj->efile.elf = elf;  in bpf_object__elf_init()
1530 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1531 		pr_warn("elf: '%s' is not a proper ELF object\n", obj->path);  in bpf_object__elf_init()
1536 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1537 		pr_warn("elf: '%s' is not a 64-bit ELF object\n", obj->path);  in bpf_object__elf_init()
1541 	obj->efile.ehdr = ehdr = elf64_getehdr(elf);  in bpf_object__elf_init()
1542 	if (!obj->efile.ehdr) {  in bpf_object__elf_init()
1543 		pr_warn("elf: failed to get ELF header from %s: %s\n", obj->path, elf_errmsg(-1));  in bpf_object__elf_init()
1544 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1548 	if (elf_getshdrstrndx(elf, &obj->efile.shstrndx)) {  in bpf_object__elf_init()
1550 			obj->path, elf_errmsg(-1));  in bpf_object__elf_init()
1551 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1556 	if (!elf_rawdata(elf_getscn(elf, obj->efile.shstrndx), NULL)) {  in bpf_object__elf_init()
1558 			obj->path, elf_errmsg(-1));  in bpf_object__elf_init()
1559 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1564 	if (ehdr->e_type != ET_REL || (ehdr->e_machine && ehdr->e_machine != EM_BPF)) {  in bpf_object__elf_init()
1565 		pr_warn("elf: %s is not a valid eBPF object file\n", obj->path);  in bpf_object__elf_init()
1566 		err = -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_init()
1579 	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2LSB)  in bpf_object__check_endianness()
1582 	if (obj->efile.ehdr->e_ident[EI_DATA] == ELFDATA2MSB)  in bpf_object__check_endianness()
1587 	pr_warn("elf: endianness mismatch in %s.\n", obj->path);  in bpf_object__check_endianness()
1588 	return -LIBBPF_ERRNO__ENDIAN;  in bpf_object__check_endianness()
1595 		pr_warn("invalid license section in %s\n", obj->path);  in bpf_object__init_license()
1596 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__init_license()
1598 	/* libbpf_strlcpy() only copies first N - 1 bytes, so size + 1 won't  in bpf_object__init_license()
1601 	libbpf_strlcpy(obj->license, data, min(size + 1, sizeof(obj->license)));  in bpf_object__init_license()
1602 	pr_debug("license of %s is %s\n", obj->path, obj->license);  in bpf_object__init_license()
1612 		pr_warn("invalid kver section in %s\n", obj->path);  in bpf_object__init_kversion()
1613 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__init_kversion()
1616 	obj->kern_version = kver;  in bpf_object__init_kversion()
1617 	pr_debug("kernel version of %s is %x\n", obj->path, obj->kern_version);  in bpf_object__init_kversion()
1635 		return -EINVAL;  in find_elf_sec_sz()
1640 		*size = data->d_size;  in find_elf_sec_sz()
1644 	return -ENOENT;  in find_elf_sec_sz()
1649 	Elf_Data *symbols = obj->efile.symbols;  in find_elf_var_sym()
1653 	for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) {  in find_elf_var_sym()
1656 		if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT)  in find_elf_var_sym()
1659 		if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL &&  in find_elf_var_sym()
1660 		    ELF64_ST_BIND(sym->st_info) != STB_WEAK)  in find_elf_var_sym()
1663 		sname = elf_sym_str(obj, sym->st_name);  in find_elf_var_sym()
1666 			return ERR_PTR(-EIO);  in find_elf_var_sym()
1672 	return ERR_PTR(-ENOENT);  in find_elf_var_sym()
1692 	fd = ensure_good_fd(sys_memfd_create("libbpf-placeholder-fd", MFD_CLOEXEC));  in create_placeholder_fd()
1694 		return -errno;  in create_placeholder_fd()
1703 	err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap,  in bpf_object__add_map()
1704 				sizeof(*obj->maps), obj->nr_maps + 1);  in bpf_object__add_map()
1708 	map = &obj->maps[obj->nr_maps++];  in bpf_object__add_map()
1709 	map->obj = obj;  in bpf_object__add_map()
1722 	map->fd = create_placeholder_fd();  in bpf_object__add_map()
1723 	if (map->fd < 0)  in bpf_object__add_map()
1724 		return ERR_PTR(map->fd);  in bpf_object__add_map()
1725 	map->inner_map_fd = -1;  in bpf_object__add_map()
1726 	map->autocreate = true;  in bpf_object__add_map()
1745 	switch (map->def.type) {  in bpf_map_mmap_sz()
1747 		return array_map_mmap_sz(map->def.value_size, map->def.max_entries);  in bpf_map_mmap_sz()
1749 		return page_sz * map->def.max_entries;  in bpf_map_mmap_sz()
1759 	if (!map->mmaped)  in bpf_map_mmap_resize()
1760 		return -EINVAL;  in bpf_map_mmap_resize()
1765 	mmaped = mmap(NULL, new_sz, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_ANONYMOUS, -1, 0);  in bpf_map_mmap_resize()
1767 		return -errno;  in bpf_map_mmap_resize()
1769 	memcpy(mmaped, map->mmaped, min(old_sz, new_sz));  in bpf_map_mmap_resize()
1770 	munmap(map->mmaped, old_sz);  in bpf_map_mmap_resize()
1771 	map->mmaped = mmaped;  in bpf_map_mmap_resize()
1796 	 * name is '.kconfig', we'll still have entirety of '.kconfig' added  in internal_map_name()
1799 	 * name will be "my_obje.kconfig".  in internal_map_name()
1805 	 * '.rodata.abracad' kernel and user-visible name.  in internal_map_name()
1814 		sfx_len = BPF_OBJ_NAME_LEN - 1;  in internal_map_name()
1820 		pfx_len = min((size_t)BPF_OBJ_NAME_LEN - sfx_len - 1, strlen(obj->name));  in internal_map_name()
1822 	snprintf(map_name, sizeof(map_name), "%.*s%.*s", pfx_len, obj->name,  in internal_map_name()
1847 	if (!map->btf_value_type_id)  in map_is_mmapable()
1850 	t = btf__type_by_id(obj->btf, map->btf_value_type_id);  in map_is_mmapable()
1856 		vt = btf__type_by_id(obj->btf, vsi->type);  in map_is_mmapable()
1860 		if (btf_var(vt)->linkage != BTF_VAR_STATIC)  in map_is_mmapable()
1880 	map->libbpf_type = type;  in bpf_object__init_internal_map()
1881 	map->sec_idx = sec_idx;  in bpf_object__init_internal_map()
1882 	map->sec_offset = 0;  in bpf_object__init_internal_map()
1883 	map->real_name = strdup(real_name);  in bpf_object__init_internal_map()
1884 	map->name = internal_map_name(obj, real_name);  in bpf_object__init_internal_map()
1885 	if (!map->real_name || !map->name) {  in bpf_object__init_internal_map()
1886 		zfree(&map->real_name);  in bpf_object__init_internal_map()
1887 		zfree(&map->name);  in bpf_object__init_internal_map()
1888 		return -ENOMEM;  in bpf_object__init_internal_map()
1891 	def = &map->def;  in bpf_object__init_internal_map()
1892 	def->type = BPF_MAP_TYPE_ARRAY;  in bpf_object__init_internal_map()
1893 	def->key_size = sizeof(int);  in bpf_object__init_internal_map()
1894 	def->value_size = data_sz;  in bpf_object__init_internal_map()
1895 	def->max_entries = 1;  in bpf_object__init_internal_map()
1896 	def->map_flags = type == LIBBPF_MAP_RODATA || type == LIBBPF_MAP_KCONFIG  in bpf_object__init_internal_map()
1903 		def->map_flags |= BPF_F_MMAPABLE;  in bpf_object__init_internal_map()
1906 		 map->name, map->sec_idx, map->sec_offset, def->map_flags);  in bpf_object__init_internal_map()
1909 	map->mmaped = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,  in bpf_object__init_internal_map()
1910 			   MAP_SHARED | MAP_ANONYMOUS, -1, 0);  in bpf_object__init_internal_map()
1911 	if (map->mmaped == MAP_FAILED) {  in bpf_object__init_internal_map()
1912 		err = -errno;  in bpf_object__init_internal_map()
1913 		map->mmaped = NULL;  in bpf_object__init_internal_map()
1915 			map->name, err);  in bpf_object__init_internal_map()
1916 		zfree(&map->real_name);  in bpf_object__init_internal_map()
1917 		zfree(&map->name);  in bpf_object__init_internal_map()
1922 		memcpy(map->mmaped, data, data_sz);  in bpf_object__init_internal_map()
1924 	pr_debug("map %td is \"%s\"\n", map - obj->maps, map->name);  in bpf_object__init_internal_map()
1935 	 * Populate obj->maps with libbpf internal maps.  in bpf_object__init_global_data_maps()
1937 	for (sec_idx = 1; sec_idx < obj->efile.sec_cnt; sec_idx++) {  in bpf_object__init_global_data_maps()
1938 		sec_desc = &obj->efile.secs[sec_idx];  in bpf_object__init_global_data_maps()
1941 		if (!sec_desc->data || sec_desc->data->d_size == 0)  in bpf_object__init_global_data_maps()
1944 		switch (sec_desc->sec_type) {  in bpf_object__init_global_data_maps()
1949 							    sec_desc->data->d_buf,  in bpf_object__init_global_data_maps()
1950 							    sec_desc->data->d_size);  in bpf_object__init_global_data_maps()
1953 			obj->has_rodata = true;  in bpf_object__init_global_data_maps()
1957 							    sec_desc->data->d_buf,  in bpf_object__init_global_data_maps()
1958 							    sec_desc->data->d_size);  in bpf_object__init_global_data_maps()
1965 							    sec_desc->data->d_size);  in bpf_object__init_global_data_maps()
1983 	for (i = 0; i < obj->nr_extern; i++) {  in find_extern_by_name()
1984 		if (strcmp(obj->externs[i].name, name) == 0)  in find_extern_by_name()
1985 			return &obj->externs[i];  in find_extern_by_name()
1990 static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val,  in set_kcfg_value_tri()  argument
1993 	switch (ext->kcfg.type) {  in set_kcfg_value_tri()
1997 				ext->name, value);  in set_kcfg_value_tri()
1998 			return -EINVAL;  in set_kcfg_value_tri()
2018 			ext->name, value);  in set_kcfg_value_tri()
2019 		return -EINVAL;  in set_kcfg_value_tri()
2021 	ext->is_set = true;  in set_kcfg_value_tri()
2025 static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val,  in set_kcfg_value_str()  argument
2030 	if (ext->kcfg.type != KCFG_CHAR_ARR) {  in set_kcfg_value_str()
2032 			ext->name, value);  in set_kcfg_value_str()
2033 		return -EINVAL;  in set_kcfg_value_str()
2037 	if (value[len - 1] != '"') {  in set_kcfg_value_str()
2039 			ext->name, value);  in set_kcfg_value_str()
2040 		return -EINVAL;  in set_kcfg_value_str()
2044 	len -= 2;  in set_kcfg_value_str()
2045 	if (len >= ext->kcfg.sz) {  in set_kcfg_value_str()
2047 			ext->name, value, len, ext->kcfg.sz - 1);  in set_kcfg_value_str()
2048 		len = ext->kcfg.sz - 1;  in set_kcfg_value_str()
2052 	ext->is_set = true;  in set_kcfg_value_str()
2064 		err = -errno;  in parse_u64()
2070 		return -EINVAL;  in parse_u64()
2075 static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v)  in is_kcfg_value_in_range()  argument
2077 	int bit_sz = ext->kcfg.sz * 8;  in is_kcfg_value_in_range()
2079 	if (ext->kcfg.sz == 8)  in is_kcfg_value_in_range()
2082 	/* Validate that value stored in u64 fits in integer of `ext->sz`  in is_kcfg_value_in_range()
2087 	 *     -2^(Y-1) <= X           <= 2^(Y-1) - 1  in is_kcfg_value_in_range()
2088 	 *            0 <= X + 2^(Y-1) <= 2^Y - 1  in is_kcfg_value_in_range()
2089 	 *            0 <= X + 2^(Y-1) <  2^Y  in is_kcfg_value_in_range()
2091 	 *  For unsigned target integer, check that all the (64 - Y) bits are  in is_kcfg_value_in_range()
2094 	if (ext->kcfg.is_signed)  in is_kcfg_value_in_range()
2095 		return v + (1ULL << (bit_sz - 1)) < (1ULL << bit_sz);  in is_kcfg_value_in_range()
2100 static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val,  in set_kcfg_value_num()  argument
2103 	if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR &&  in set_kcfg_value_num()
2104 	    ext->kcfg.type != KCFG_BOOL) {  in set_kcfg_value_num()
2106 			ext->name, (unsigned long long)value);  in set_kcfg_value_num()
2107 		return -EINVAL;  in set_kcfg_value_num()
2109 	if (ext->kcfg.type == KCFG_BOOL && value > 1) {  in set_kcfg_value_num()
2111 			ext->name, (unsigned long long)value);  in set_kcfg_value_num()
2112 		return -EINVAL;  in set_kcfg_value_num()
2115 	if (!is_kcfg_value_in_range(ext, value)) {  in set_kcfg_value_num()
2117 			ext->name, (unsigned long long)value, ext->kcfg.sz);  in set_kcfg_value_num()
2118 		return -ERANGE;  in set_kcfg_value_num()
2120 	switch (ext->kcfg.sz) {  in set_kcfg_value_num()
2134 		return -EINVAL;  in set_kcfg_value_num()
2136 	ext->is_set = true;  in set_kcfg_value_num()
2143 	struct extern_desc *ext;  in bpf_object__process_kconfig_line()  local
2155 		return -EINVAL;  in bpf_object__process_kconfig_line()
2160 	if (buf[len - 1] == '\n')  in bpf_object__process_kconfig_line()
2161 		buf[len - 1] = '\0';  in bpf_object__process_kconfig_line()
2167 		return -EINVAL;  in bpf_object__process_kconfig_line()
2170 	ext = find_extern_by_name(obj, buf);  in bpf_object__process_kconfig_line()
2171 	if (!ext || ext->is_set)  in bpf_object__process_kconfig_line()
2174 	ext_val = data + ext->kcfg.data_off;  in bpf_object__process_kconfig_line()
2179 		err = set_kcfg_value_tri(ext, ext_val, *value);  in bpf_object__process_kconfig_line()
2182 		err = set_kcfg_value_str(ext, ext_val, value);  in bpf_object__process_kconfig_line()
2188 			pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value);  in bpf_object__process_kconfig_line()
2191 		if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) {  in bpf_object__process_kconfig_line()
2192 			pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value);  in bpf_object__process_kconfig_line()
2193 			return -EINVAL;  in bpf_object__process_kconfig_line()
2195 		err = set_kcfg_value_num(ext, ext_val, num);  in bpf_object__process_kconfig_line()
2200 	pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value);  in bpf_object__process_kconfig_line()
2212 	len = snprintf(buf, PATH_MAX, "/boot/config-%s", uts.release);  in bpf_object__read_kconfig_file()
2214 		return -EINVAL;  in bpf_object__read_kconfig_file()
2216 		return -ENAMETOOLONG;  in bpf_object__read_kconfig_file()
2224 		pr_warn("failed to open system Kconfig\n");  in bpf_object__read_kconfig_file()
2225 		return -ENOENT;  in bpf_object__read_kconfig_file()
2231 			pr_warn("error parsing system Kconfig line '%s': %d\n",  in bpf_object__read_kconfig_file()
2251 		err = -errno;  in bpf_object__read_kconfig_mem()
2252 		pr_warn("failed to open in-memory Kconfig: %d\n", err);  in bpf_object__read_kconfig_mem()
2259 			pr_warn("error parsing in-memory Kconfig line '%s': %d\n",  in bpf_object__read_kconfig_mem()
2271 	struct extern_desc *last_ext = NULL, *ext;  in bpf_object__init_kconfig_map()  local
2275 	for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__init_kconfig_map()
2276 		ext = &obj->externs[i];  in bpf_object__init_kconfig_map()
2277 		if (ext->type == EXT_KCFG)  in bpf_object__init_kconfig_map()
2278 			last_ext = ext;  in bpf_object__init_kconfig_map()
2284 	map_sz = last_ext->kcfg.data_off + last_ext->kcfg.sz;  in bpf_object__init_kconfig_map()
2286 					    ".kconfig", obj->efile.symbols_shndx,  in bpf_object__init_kconfig_map()
2291 	obj->kconfig_map_idx = obj->nr_maps - 1;  in bpf_object__init_kconfig_map()
2306 			*res_id = t->type;  in skip_mods_and_typedefs()
2307 		t = btf__type_by_id(btf, t->type);  in skip_mods_and_typedefs()
2322 	t = skip_mods_and_typedefs(btf, t->type, res_id);  in resolve_func_ptr()
2369 	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);  in get_map_field_int()
2370 	const char *name = btf__name_by_offset(btf, m->name_off);  in get_map_field_int()
2380 	arr_t = btf__type_by_id(btf, t->type);  in get_map_field_int()
2383 			map_name, name, t->type);  in get_map_field_int()
2392 	*res = arr_info->nelems;  in get_map_field_int()
2399 	const struct btf_type *t = skip_mods_and_typedefs(btf, m->type, NULL);  in get_map_field_long()
2400 	const char *name = btf__name_by_offset(btf, m->name_off);  in get_map_field_long()
2427 		*res = e->val;  in get_map_field_long()
2442 		return -EINVAL;  in pathname_concat()
2444 		return -ENAMETOOLONG;  in pathname_concat()
2483 		const char *name = btf__name_by_offset(btf, m->name_off);  in parse_btf_map_def()
2487 			return -EINVAL;  in parse_btf_map_def()
2490 			if (!get_map_field_int(map_name, btf, m, &map_def->map_type))  in parse_btf_map_def()
2491 				return -EINVAL;  in parse_btf_map_def()
2492 			map_def->parts |= MAP_DEF_MAP_TYPE;  in parse_btf_map_def()
2494 			if (!get_map_field_int(map_name, btf, m, &map_def->max_entries))  in parse_btf_map_def()
2495 				return -EINVAL;  in parse_btf_map_def()
2496 			map_def->parts |= MAP_DEF_MAX_ENTRIES;  in parse_btf_map_def()
2498 			if (!get_map_field_int(map_name, btf, m, &map_def->map_flags))  in parse_btf_map_def()
2499 				return -EINVAL;  in parse_btf_map_def()
2500 			map_def->parts |= MAP_DEF_MAP_FLAGS;  in parse_btf_map_def()
2502 			if (!get_map_field_int(map_name, btf, m, &map_def->numa_node))  in parse_btf_map_def()
2503 				return -EINVAL;  in parse_btf_map_def()
2504 			map_def->parts |= MAP_DEF_NUMA_NODE;  in parse_btf_map_def()
2509 				return -EINVAL;  in parse_btf_map_def()
2510 			if (map_def->key_size && map_def->key_size != sz) {  in parse_btf_map_def()
2512 					map_name, map_def->key_size, sz);  in parse_btf_map_def()
2513 				return -EINVAL;  in parse_btf_map_def()
2515 			map_def->key_size = sz;  in parse_btf_map_def()
2516 			map_def->parts |= MAP_DEF_KEY_SIZE;  in parse_btf_map_def()
2520 			t = btf__type_by_id(btf, m->type);  in parse_btf_map_def()
2523 					map_name, m->type);  in parse_btf_map_def()
2524 				return -EINVAL;  in parse_btf_map_def()
2529 				return -EINVAL;  in parse_btf_map_def()
2531 			sz = btf__resolve_size(btf, t->type);  in parse_btf_map_def()
2534 					map_name, t->type, (ssize_t)sz);  in parse_btf_map_def()
2537 			if (map_def->key_size && map_def->key_size != sz) {  in parse_btf_map_def()
2539 					map_name, map_def->key_size, (ssize_t)sz);  in parse_btf_map_def()
2540 				return -EINVAL;  in parse_btf_map_def()
2542 			map_def->key_size = sz;  in parse_btf_map_def()
2543 			map_def->key_type_id = t->type;  in parse_btf_map_def()
2544 			map_def->parts |= MAP_DEF_KEY_SIZE | MAP_DEF_KEY_TYPE;  in parse_btf_map_def()
2549 				return -EINVAL;  in parse_btf_map_def()
2550 			if (map_def->value_size && map_def->value_size != sz) {  in parse_btf_map_def()
2552 					map_name, map_def->value_size, sz);  in parse_btf_map_def()
2553 				return -EINVAL;  in parse_btf_map_def()
2555 			map_def->value_size = sz;  in parse_btf_map_def()
2556 			map_def->parts |= MAP_DEF_VALUE_SIZE;  in parse_btf_map_def()
2560 			t = btf__type_by_id(btf, m->type);  in parse_btf_map_def()
2563 					map_name, m->type);  in parse_btf_map_def()
2564 				return -EINVAL;  in parse_btf_map_def()
2569 				return -EINVAL;  in parse_btf_map_def()
2571 			sz = btf__resolve_size(btf, t->type);  in parse_btf_map_def()
2574 					map_name, t->type, (ssize_t)sz);  in parse_btf_map_def()
2577 			if (map_def->value_size && map_def->value_size != sz) {  in parse_btf_map_def()
2579 					map_name, map_def->value_size, (ssize_t)sz);  in parse_btf_map_def()
2580 				return -EINVAL;  in parse_btf_map_def()
2582 			map_def->value_size = sz;  in parse_btf_map_def()
2583 			map_def->value_type_id = t->type;  in parse_btf_map_def()
2584 			map_def->parts |= MAP_DEF_VALUE_SIZE | MAP_DEF_VALUE_TYPE;  in parse_btf_map_def()
2587 			bool is_map_in_map = bpf_map_type__is_map_in_map(map_def->map_type);  in parse_btf_map_def()
2588 			bool is_prog_array = map_def->map_type == BPF_MAP_TYPE_PROG_ARRAY;  in parse_btf_map_def()
2589 			const char *desc = is_map_in_map ? "map-in-map inner" : "prog-array value";  in parse_btf_map_def()
2594 				pr_warn("map '%s': multi-level inner maps not supported.\n",  in parse_btf_map_def()
2596 				return -ENOTSUP;  in parse_btf_map_def()
2598 			if (i != vlen - 1) {  in parse_btf_map_def()
2601 				return -EINVAL;  in parse_btf_map_def()
2604 				pr_warn("map '%s': should be map-in-map or prog-array.\n",  in parse_btf_map_def()
2606 				return -ENOTSUP;  in parse_btf_map_def()
2608 			if (map_def->value_size && map_def->value_size != 4) {  in parse_btf_map_def()
2610 					map_name, map_def->value_size);  in parse_btf_map_def()
2611 				return -EINVAL;  in parse_btf_map_def()
2613 			map_def->value_size = 4;  in parse_btf_map_def()
2614 			t = btf__type_by_id(btf, m->type);  in parse_btf_map_def()
2617 					map_name, desc, m->type);  in parse_btf_map_def()
2618 				return -EINVAL;  in parse_btf_map_def()
2620 			if (!btf_is_array(t) || btf_array(t)->nelems) {  in parse_btf_map_def()
2621 				pr_warn("map '%s': %s spec is not a zero-sized array.\n",  in parse_btf_map_def()
2623 				return -EINVAL;  in parse_btf_map_def()
2625 			t = skip_mods_and_typedefs(btf, btf_array(t)->type, NULL);  in parse_btf_map_def()
2629 				return -EINVAL;  in parse_btf_map_def()
2631 			t = skip_mods_and_typedefs(btf, t->type, NULL);  in parse_btf_map_def()
2634 					pr_warn("map '%s': prog-array value def is of unexpected kind %s.\n",  in parse_btf_map_def()
2636 					return -EINVAL;  in parse_btf_map_def()
2641 				pr_warn("map '%s': map-in-map inner def is of unexpected kind %s.\n",  in parse_btf_map_def()
2643 				return -EINVAL;  in parse_btf_map_def()
2651 			map_def->parts |= MAP_DEF_INNER_MAP;  in parse_btf_map_def()
2657 				return -EINVAL;  in parse_btf_map_def()
2660 				return -EINVAL;  in parse_btf_map_def()
2664 				return -EINVAL;  in parse_btf_map_def()
2666 			map_def->pinning = val;  in parse_btf_map_def()
2667 			map_def->parts |= MAP_DEF_PINNING;  in parse_btf_map_def()
2672 				return -EINVAL;  in parse_btf_map_def()
2673 			map_def->map_extra = map_extra;  in parse_btf_map_def()
2674 			map_def->parts |= MAP_DEF_MAP_EXTRA;  in parse_btf_map_def()
2678 				return -ENOTSUP;  in parse_btf_map_def()
2684 	if (map_def->map_type == BPF_MAP_TYPE_UNSPEC) {  in parse_btf_map_def()
2686 		return -EINVAL;  in parse_btf_map_def()
2701 	 * a power-of-2 multiple of kernel's page size. If user diligently  in adjust_ringbuf_sz()
2708 	 * user-set size to satisfy both user size request and kernel  in adjust_ringbuf_sz()
2717 	 * very close to UINT_MAX but is not a power-of-2 multiple of  in adjust_ringbuf_sz()
2725 	return map->def.type == BPF_MAP_TYPE_RINGBUF ||  in map_is_ringbuf()
2726 	       map->def.type == BPF_MAP_TYPE_USER_RINGBUF;  in map_is_ringbuf()
2731 	map->def.type = def->map_type;  in fill_map_from_def()
2732 	map->def.key_size = def->key_size;  in fill_map_from_def()
2733 	map->def.value_size = def->value_size;  in fill_map_from_def()
2734 	map->def.max_entries = def->max_entries;  in fill_map_from_def()
2735 	map->def.map_flags = def->map_flags;  in fill_map_from_def()
2736 	map->map_extra = def->map_extra;  in fill_map_from_def()
2738 	map->numa_node = def->numa_node;  in fill_map_from_def()
2739 	map->btf_key_type_id = def->key_type_id;  in fill_map_from_def()
2740 	map->btf_value_type_id = def->value_type_id;  in fill_map_from_def()
2742 	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */  in fill_map_from_def()
2744 		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);  in fill_map_from_def()
2746 	if (def->parts & MAP_DEF_MAP_TYPE)  in fill_map_from_def()
2747 		pr_debug("map '%s': found type = %u.\n", map->name, def->map_type);  in fill_map_from_def()
2749 	if (def->parts & MAP_DEF_KEY_TYPE)  in fill_map_from_def()
2751 			 map->name, def->key_type_id, def->key_size);  in fill_map_from_def()
2752 	else if (def->parts & MAP_DEF_KEY_SIZE)  in fill_map_from_def()
2753 		pr_debug("map '%s': found key_size = %u.\n", map->name, def->key_size);  in fill_map_from_def()
2755 	if (def->parts & MAP_DEF_VALUE_TYPE)  in fill_map_from_def()
2757 			 map->name, def->value_type_id, def->value_size);  in fill_map_from_def()
2758 	else if (def->parts & MAP_DEF_VALUE_SIZE)  in fill_map_from_def()
2759 		pr_debug("map '%s': found value_size = %u.\n", map->name, def->value_size);  in fill_map_from_def()
2761 	if (def->parts & MAP_DEF_MAX_ENTRIES)  in fill_map_from_def()
2762 		pr_debug("map '%s': found max_entries = %u.\n", map->name, def->max_entries);  in fill_map_from_def()
2763 	if (def->parts & MAP_DEF_MAP_FLAGS)  in fill_map_from_def()
2764 		pr_debug("map '%s': found map_flags = 0x%x.\n", map->name, def->map_flags);  in fill_map_from_def()
2765 	if (def->parts & MAP_DEF_MAP_EXTRA)  in fill_map_from_def()
2766 		pr_debug("map '%s': found map_extra = 0x%llx.\n", map->name,  in fill_map_from_def()
2767 			 (unsigned long long)def->map_extra);  in fill_map_from_def()
2768 	if (def->parts & MAP_DEF_PINNING)  in fill_map_from_def()
2769 		pr_debug("map '%s': found pinning = %u.\n", map->name, def->pinning);  in fill_map_from_def()
2770 	if (def->parts & MAP_DEF_NUMA_NODE)  in fill_map_from_def()
2771 		pr_debug("map '%s': found numa_node = %u.\n", map->name, def->numa_node);  in fill_map_from_def()
2773 	if (def->parts & MAP_DEF_INNER_MAP)  in fill_map_from_def()
2774 		pr_debug("map '%s': found inner map definition.\n", map->name);  in fill_map_from_def()
2802 	var = btf__type_by_id(obj->btf, vi->type);  in bpf_object__init_user_btf_map()
2804 	map_name = btf__name_by_offset(obj->btf, var->name_off);  in bpf_object__init_user_btf_map()
2808 		return -EINVAL;  in bpf_object__init_user_btf_map()
2810 	if ((__u64)vi->offset + vi->size > data->d_size) {  in bpf_object__init_user_btf_map()
2812 		return -EINVAL;  in bpf_object__init_user_btf_map()
2817 		return -EINVAL;  in bpf_object__init_user_btf_map()
2819 	if (var_extra->linkage != BTF_VAR_GLOBAL_ALLOCATED) {  in bpf_object__init_user_btf_map()
2821 			map_name, btf_var_linkage_str(var_extra->linkage));  in bpf_object__init_user_btf_map()
2822 		return -EOPNOTSUPP;  in bpf_object__init_user_btf_map()
2825 	def = skip_mods_and_typedefs(obj->btf, var->type, NULL);  in bpf_object__init_user_btf_map()
2829 		return -EINVAL;  in bpf_object__init_user_btf_map()
2831 	if (def->size > vi->size) {  in bpf_object__init_user_btf_map()
2833 		return -EINVAL;  in bpf_object__init_user_btf_map()
2839 	map->name = strdup(map_name);  in bpf_object__init_user_btf_map()
2840 	if (!map->name) {  in bpf_object__init_user_btf_map()
2842 		return -ENOMEM;  in bpf_object__init_user_btf_map()
2844 	map->libbpf_type = LIBBPF_MAP_UNSPEC;  in bpf_object__init_user_btf_map()
2845 	map->def.type = BPF_MAP_TYPE_UNSPEC;  in bpf_object__init_user_btf_map()
2846 	map->sec_idx = sec_idx;  in bpf_object__init_user_btf_map()
2847 	map->sec_offset = vi->offset;  in bpf_object__init_user_btf_map()
2848 	map->btf_var_idx = var_idx;  in bpf_object__init_user_btf_map()
2850 		 map_name, map->sec_idx, map->sec_offset);  in bpf_object__init_user_btf_map()
2852 	err = parse_btf_map_def(map->name, obj->btf, def, strict, &map_def, &inner_def);  in bpf_object__init_user_btf_map()
2861 			pr_warn("map '%s': couldn't build pin path.\n", map->name);  in bpf_object__init_user_btf_map()
2867 		map->inner_map = calloc(1, sizeof(*map->inner_map));  in bpf_object__init_user_btf_map()
2868 		if (!map->inner_map)  in bpf_object__init_user_btf_map()
2869 			return -ENOMEM;  in bpf_object__init_user_btf_map()
2870 		map->inner_map->fd = create_placeholder_fd();  in bpf_object__init_user_btf_map()
2871 		if (map->inner_map->fd < 0)  in bpf_object__init_user_btf_map()
2872 			return map->inner_map->fd;  in bpf_object__init_user_btf_map()
2873 		map->inner_map->sec_idx = sec_idx;  in bpf_object__init_user_btf_map()
2874 		map->inner_map->name = malloc(strlen(map_name) + sizeof(".inner") + 1);  in bpf_object__init_user_btf_map()
2875 		if (!map->inner_map->name)  in bpf_object__init_user_btf_map()
2876 			return -ENOMEM;  in bpf_object__init_user_btf_map()
2877 		sprintf(map->inner_map->name, "%s.inner", map_name);  in bpf_object__init_user_btf_map()
2879 		fill_map_from_def(map->inner_map, &inner_def);  in bpf_object__init_user_btf_map()
2896 	mmap_sz = bpf_map_mmap_sz(obj->arena_map);  in init_arena_map_data()
2900 		return -E2BIG;  in init_arena_map_data()
2903 	obj->arena_data = malloc(data_sz);  in init_arena_map_data()
2904 	if (!obj->arena_data)  in init_arena_map_data()
2905 		return -ENOMEM;  in init_arena_map_data()
2906 	memcpy(obj->arena_data, data, data_sz);  in init_arena_map_data()
2907 	obj->arena_data_sz = data_sz;  in init_arena_map_data()
2910 	map->mmaped = obj->arena_data;  in init_arena_map_data()
2925 	if (obj->efile.btf_maps_shndx < 0)  in bpf_object__init_user_btf_maps()
2928 	scn = elf_sec_by_idx(obj, obj->efile.btf_maps_shndx);  in bpf_object__init_user_btf_maps()
2932 			MAPS_ELF_SEC, obj->path);  in bpf_object__init_user_btf_maps()
2933 		return -EINVAL;  in bpf_object__init_user_btf_maps()
2936 	nr_types = btf__type_cnt(obj->btf);  in bpf_object__init_user_btf_maps()
2938 		t = btf__type_by_id(obj->btf, i);  in bpf_object__init_user_btf_maps()
2941 		name = btf__name_by_offset(obj->btf, t->name_off);  in bpf_object__init_user_btf_maps()
2944 			obj->efile.btf_maps_sec_btf_id = i;  in bpf_object__init_user_btf_maps()
2951 		return -ENOENT;  in bpf_object__init_user_btf_maps()
2957 						    obj->efile.btf_maps_shndx,  in bpf_object__init_user_btf_maps()
2964 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object__init_user_btf_maps()
2965 		struct bpf_map *map = &obj->maps[i];  in bpf_object__init_user_btf_maps()
2967 		if (map->def.type != BPF_MAP_TYPE_ARENA)  in bpf_object__init_user_btf_maps()
2970 		if (obj->arena_map) {  in bpf_object__init_user_btf_maps()
2972 				map->name, obj->arena_map->name);  in bpf_object__init_user_btf_maps()
2973 			return -EINVAL;  in bpf_object__init_user_btf_maps()
2975 		obj->arena_map = map;  in bpf_object__init_user_btf_maps()
2977 		if (obj->efile.arena_data) {  in bpf_object__init_user_btf_maps()
2978 			err = init_arena_map_data(obj, map, ARENA_SEC, obj->efile.arena_data_shndx,  in bpf_object__init_user_btf_maps()
2979 						  obj->efile.arena_data->d_buf,  in bpf_object__init_user_btf_maps()
2980 						  obj->efile.arena_data->d_size);  in bpf_object__init_user_btf_maps()
2985 	if (obj->efile.arena_data && !obj->arena_map) {  in bpf_object__init_user_btf_maps()
2988 		return -ENOENT;  in bpf_object__init_user_btf_maps()
3020 	return sh->sh_flags & SHF_EXECINSTR;  in section_have_execinstr()
3062 			t->info = BTF_INFO_ENC(BTF_KIND_INT, 0, 0);  in bpf_object__sanitize_btf()
3068 			t->size = 1;  in bpf_object__sanitize_btf()
3077 			name = (char *)btf__name_by_offset(btf, t->name_off);  in bpf_object__sanitize_btf()
3085 			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, vlen);  in bpf_object__sanitize_btf()
3088 				m->offset = v->offset * 8;  in bpf_object__sanitize_btf()
3089 				m->type = v->type;  in bpf_object__sanitize_btf()
3091 				vt = (void *)btf__type_by_id(btf, v->type);  in bpf_object__sanitize_btf()
3092 				m->name_off = vt->name_off;  in bpf_object__sanitize_btf()
3095 			   starts_with_qmark(btf__name_by_offset(btf, t->name_off))) {  in bpf_object__sanitize_btf()
3099 			name = (char *)btf__name_by_offset(btf, t->name_off);  in bpf_object__sanitize_btf()
3105 			t->info = BTF_INFO_ENC(BTF_KIND_ENUM, 0, vlen);  in bpf_object__sanitize_btf()
3106 			t->size = sizeof(__u32); /* kernel enforced */  in bpf_object__sanitize_btf()
3109 			t->info = BTF_INFO_ENC(BTF_KIND_TYPEDEF, 0, 0);  in bpf_object__sanitize_btf()
3112 			t->info = BTF_INFO_ENC(BTF_KIND_FUNC, 0, 0);  in bpf_object__sanitize_btf()
3114 			/* replace FLOAT with an equally-sized empty STRUCT;  in bpf_object__sanitize_btf()
3118 			t->name_off = 0;  in bpf_object__sanitize_btf()
3119 			t->info = BTF_INFO_ENC(BTF_KIND_STRUCT, 0, 0);  in bpf_object__sanitize_btf()
3122 			t->name_off = 0;  in bpf_object__sanitize_btf()
3123 			t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0);  in bpf_object__sanitize_btf()
3126 			t->info = btf_type_info(btf_kind(t), btf_vlen(t), false);  in bpf_object__sanitize_btf()
3141 			t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen);  in bpf_object__sanitize_btf()
3143 				m->type = enum64_placeholder_id;  in bpf_object__sanitize_btf()
3144 				m->offset = 0;  in bpf_object__sanitize_btf()
3154 	return obj->efile.btf_maps_shndx >= 0 ||  in libbpf_needs_btf()
3155 	       obj->efile.has_st_ops ||  in libbpf_needs_btf()
3156 	       obj->nr_extern > 0;  in libbpf_needs_btf()
3161 	return obj->efile.has_st_ops;  in kernel_needs_btf()
3168 	int err = -ENOENT;  in bpf_object__init_btf()
3171 		obj->btf = btf__new(btf_data->d_buf, btf_data->d_size);  in bpf_object__init_btf()
3172 		err = libbpf_get_error(obj->btf);  in bpf_object__init_btf()
3174 			obj->btf = NULL;  in bpf_object__init_btf()
3178 		/* enforce 8-byte pointers for BPF-targeted BTFs */  in bpf_object__init_btf()
3179 		btf__set_pointer_size(obj->btf, 8);  in bpf_object__init_btf()
3185 		if (!obj->btf) {  in bpf_object__init_btf()
3190 		obj->btf_ext = btf_ext__new(btf_ext_data->d_buf, btf_ext_data->d_size);  in bpf_object__init_btf()
3191 		err = libbpf_get_error(obj->btf_ext);  in bpf_object__init_btf()
3195 			obj->btf_ext = NULL;  in bpf_object__init_btf()
3199 		/* setup .BTF.ext to ELF section mapping */  in bpf_object__init_btf()
3200 		ext_segs[0] = &obj->btf_ext->func_info;  in bpf_object__init_btf()
3201 		ext_segs[1] = &obj->btf_ext->line_info;  in bpf_object__init_btf()
3202 		ext_segs[2] = &obj->btf_ext->core_relo_info;  in bpf_object__init_btf()
3209 			if (seg->sec_cnt == 0)  in bpf_object__init_btf()
3212 			seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs));  in bpf_object__init_btf()
3213 			if (!seg->sec_idxs) {  in bpf_object__init_btf()
3214 				err = -ENOMEM;  in bpf_object__init_btf()
3225 				sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);  in bpf_object__init_btf()
3232 				seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn);  in bpf_object__init_btf()
3249 	return a->offset - b->offset;  in compare_vsi_off()
3256 	const char *sec_name = btf__name_by_offset(btf, t->name_off);  in btf_fixup_datasec()
3263 		return -ENOENT;  in btf_fixup_datasec()
3266 	/* Extern-backing datasecs (.ksyms, .kconfig) have their size and  in btf_fixup_datasec()
3278 	 * to be optional. But the STV_HIDDEN handling is non-optional for any  in btf_fixup_datasec()
3279 	 * non-extern DATASEC, so the variable fixup loop below handles both  in btf_fixup_datasec()
3280 	 * functions at the same time, paying the cost of BTF VAR <-> ELF  in btf_fixup_datasec()
3283 	if (t->size == 0) {  in btf_fixup_datasec()
3288 			return -ENOENT;  in btf_fixup_datasec()
3291 		t->size = size;  in btf_fixup_datasec()
3301 		t_var = btf__type_by_id(btf, vsi->type);  in btf_fixup_datasec()
3303 			pr_debug("sec '%s': unexpected non-VAR type found\n", sec_name);  in btf_fixup_datasec()
3304 			return -EINVAL;  in btf_fixup_datasec()
3308 		if (var->linkage == BTF_VAR_STATIC || var->linkage == BTF_VAR_GLOBAL_EXTERN)  in btf_fixup_datasec()
3311 		var_name = btf__name_by_offset(btf, t_var->name_off);  in btf_fixup_datasec()
3315 			return -ENOENT;  in btf_fixup_datasec()
3322 			return -ENOENT;  in btf_fixup_datasec()
3326 			vsi->offset = sym->st_value;  in btf_fixup_datasec()
3335 		if (ELF64_ST_VISIBILITY(sym->st_other) == STV_HIDDEN  in btf_fixup_datasec()
3336 		    || ELF64_ST_VISIBILITY(sym->st_other) == STV_INTERNAL)  in btf_fixup_datasec()
3337 			var->linkage = BTF_VAR_STATIC;  in btf_fixup_datasec()
3349 	if (!obj->btf)  in bpf_object_fixup_btf()
3352 	n = btf__type_cnt(obj->btf);  in bpf_object_fixup_btf()
3354 		struct btf_type *t = btf_type_by_id(obj->btf, i);  in bpf_object_fixup_btf()
3362 			err = btf_fixup_datasec(obj, obj->btf, t);  in bpf_object_fixup_btf()
3373 	if (prog->type == BPF_PROG_TYPE_STRUCT_OPS ||  in prog_needs_vmlinux_btf()
3374 	    prog->type == BPF_PROG_TYPE_LSM)  in prog_needs_vmlinux_btf()
3380 	if (prog->type == BPF_PROG_TYPE_TRACING && !prog->attach_prog_fd)  in prog_needs_vmlinux_btf()
3397 	/* CO-RE relocations need kernel BTF, only when btf_custom_path  in obj_needs_vmlinux_btf()
3400 	if (obj->btf_ext && obj->btf_ext->core_relo_info.len && !obj->btf_custom_path)  in obj_needs_vmlinux_btf()
3404 	for (i = 0; i < obj->nr_extern; i++) {  in obj_needs_vmlinux_btf()
3405 		const struct extern_desc *ext;  in obj_needs_vmlinux_btf()  local
3407 		ext = &obj->externs[i];  in obj_needs_vmlinux_btf()
3408 		if (ext->type == EXT_KSYM && ext->ksym.type_id)  in obj_needs_vmlinux_btf()
3413 		if (!prog->autoload)  in obj_needs_vmlinux_btf()
3432 	if (obj->btf_vmlinux || obj->gen_loader)  in bpf_object__load_vmlinux_btf()
3438 	obj->btf_vmlinux = btf__load_vmlinux_btf();  in bpf_object__load_vmlinux_btf()
3439 	err = libbpf_get_error(obj->btf_vmlinux);  in bpf_object__load_vmlinux_btf()
3442 		obj->btf_vmlinux = NULL;  in bpf_object__load_vmlinux_btf()
3450 	struct btf *kern_btf = obj->btf;  in bpf_object__sanitize_and_load_btf()
3454 	if (!obj->btf)  in bpf_object__sanitize_and_load_btf()
3459 			err = -EOPNOTSUPP;  in bpf_object__sanitize_and_load_btf()
3474 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__sanitize_and_load_btf()
3475 		struct bpf_program *prog = &obj->programs[i];  in bpf_object__sanitize_and_load_btf()
3480 		if (!prog->mark_btf_static || !prog_is_subprog(obj, prog))  in bpf_object__sanitize_and_load_btf()
3483 		n = btf__type_cnt(obj->btf);  in bpf_object__sanitize_and_load_btf()
3485 			t = btf_type_by_id(obj->btf, j);  in bpf_object__sanitize_and_load_btf()
3489 			name = btf__str_by_offset(obj->btf, t->name_off);  in bpf_object__sanitize_and_load_btf()
3490 			if (strcmp(name, prog->name) != 0)  in bpf_object__sanitize_and_load_btf()
3493 			t->info = btf_type_info(BTF_KIND_FUNC, BTF_FUNC_STATIC, 0);  in bpf_object__sanitize_and_load_btf()
3504 		raw_data = btf__raw_data(obj->btf, &sz);  in bpf_object__sanitize_and_load_btf()
3510 		/* enforce 8-byte pointers for BPF-targeted BTFs */  in bpf_object__sanitize_and_load_btf()
3511 		btf__set_pointer_size(obj->btf, 8);  in bpf_object__sanitize_and_load_btf()
3517 	if (obj->gen_loader) {  in bpf_object__sanitize_and_load_btf()
3522 			return -ENOMEM;  in bpf_object__sanitize_and_load_btf()
3523 		bpf_gen__load_btf(obj->gen_loader, raw_data, raw_size);  in bpf_object__sanitize_and_load_btf()
3525 		 * This fd == 0 will not be used with any syscall and will be reset to -1 eventually.  in bpf_object__sanitize_and_load_btf()
3530 		err = btf_load_into_kernel(kern_btf, obj->log_buf, obj->log_size,  in bpf_object__sanitize_and_load_btf()
3531 					   obj->log_level ? 1 : 0, obj->token_fd);  in bpf_object__sanitize_and_load_btf()
3536 			btf__set_fd(obj->btf, btf__fd(kern_btf));  in bpf_object__sanitize_and_load_btf()
3537 			btf__set_fd(kern_btf, -1);  in bpf_object__sanitize_and_load_btf()
3557 	name = elf_strptr(obj->efile.elf, obj->efile.strtabidx, off);  in elf_sym_str()
3560 			off, obj->path, elf_errmsg(-1));  in elf_sym_str()
3571 	name = elf_strptr(obj->efile.elf, obj->efile.shstrndx, off);  in elf_sec_str()
3574 			off, obj->path, elf_errmsg(-1));  in elf_sec_str()
3585 	scn = elf_getscn(obj->efile.elf, idx);  in elf_sec_by_idx()
3588 			idx, obj->path, elf_errmsg(-1));  in elf_sec_by_idx()
3597 	Elf *elf = obj->efile.elf;  in elf_sec_by_name()
3623 			elf_ndxscn(scn), obj->path, elf_errmsg(-1));  in elf_sec_hdr()
3642 	name = elf_sec_str(obj, sh->sh_name);  in elf_sec_name()
3645 			elf_ndxscn(scn), obj->path, elf_errmsg(-1));  in elf_sec_name()
3663 			obj->path, elf_errmsg(-1));  in elf_sec_data()
3672 	if (idx >= obj->efile.symbols->d_size / sizeof(Elf64_Sym))  in elf_sym_by_idx()
3675 	return (Elf64_Sym *)obj->efile.symbols->d_buf + idx;  in elf_sym_by_idx()
3680 	if (idx >= data->d_size / sizeof(Elf64_Rel))  in elf_rel_by_idx()
3683 	return (Elf64_Rel *)data->d_buf + idx;  in elf_rel_by_idx()
3695 	if (hdr->sh_type == SHT_STRTAB)  in ignore_elf_section()
3699 	if (hdr->sh_type == SHT_LLVM_ADDRSIG)  in ignore_elf_section()
3703 	if (hdr->sh_type == SHT_PROGBITS && hdr->sh_size == 0 &&  in ignore_elf_section()
3712 		name += sizeof(".rel") - 1;  in ignore_elf_section()
3717 		/* .BTF and .BTF.ext don't need relocations */  in ignore_elf_section()
3731 	if (a->sec_idx != b->sec_idx)  in cmp_progs()
3732 		return a->sec_idx < b->sec_idx ? -1 : 1;  in cmp_progs()
3735 	return a->sec_insn_off < b->sec_insn_off ? -1 : 1;  in cmp_progs()
3741 	Elf *elf = obj->efile.elf;  in bpf_object__elf_collect()
3750 	/* ELF section indices are 0-based, but sec #0 is special "invalid"  in bpf_object__elf_collect()
3755 	if (elf_getshdrnum(obj->efile.elf, &obj->efile.sec_cnt)) {  in bpf_object__elf_collect()
3757 			obj->path, elf_errmsg(-1));  in bpf_object__elf_collect()
3758 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3760 	obj->efile.secs = calloc(obj->efile.sec_cnt, sizeof(*obj->efile.secs));  in bpf_object__elf_collect()
3761 	if (!obj->efile.secs)  in bpf_object__elf_collect()
3762 		return -ENOMEM;  in bpf_object__elf_collect()
3771 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3773 		if (sh->sh_type == SHT_SYMTAB) {  in bpf_object__elf_collect()
3774 			if (obj->efile.symbols) {  in bpf_object__elf_collect()
3775 				pr_warn("elf: multiple symbol tables in %s\n", obj->path);  in bpf_object__elf_collect()
3776 				return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3781 				return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3785 			obj->efile.symbols = data;  in bpf_object__elf_collect()
3786 			obj->efile.symbols_shndx = idx;  in bpf_object__elf_collect()
3787 			obj->efile.strtabidx = sh->sh_link;  in bpf_object__elf_collect()
3791 	if (!obj->efile.symbols) {  in bpf_object__elf_collect()
3793 			obj->path);  in bpf_object__elf_collect()
3794 		return -ENOENT;  in bpf_object__elf_collect()
3800 		sec_desc = &obj->efile.secs[idx];  in bpf_object__elf_collect()
3804 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3806 		name = elf_sec_str(obj, sh->sh_name);  in bpf_object__elf_collect()
3808 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3815 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3818 			 idx, name, (unsigned long)data->d_size,  in bpf_object__elf_collect()
3819 			 (int)sh->sh_link, (unsigned long)sh->sh_flags,  in bpf_object__elf_collect()
3820 			 (int)sh->sh_type);  in bpf_object__elf_collect()
3823 			err = bpf_object__init_license(obj, data->d_buf, data->d_size);  in bpf_object__elf_collect()
3827 			err = bpf_object__init_kversion(obj, data->d_buf, data->d_size);  in bpf_object__elf_collect()
3832 			return -ENOTSUP;  in bpf_object__elf_collect()
3834 			obj->efile.btf_maps_shndx = idx;  in bpf_object__elf_collect()
3836 			if (sh->sh_type != SHT_PROGBITS)  in bpf_object__elf_collect()
3837 				return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3840 			if (sh->sh_type != SHT_PROGBITS)  in bpf_object__elf_collect()
3841 				return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3843 		} else if (sh->sh_type == SHT_SYMTAB) {  in bpf_object__elf_collect()
3845 		} else if (sh->sh_type == SHT_PROGBITS && data->d_size > 0) {  in bpf_object__elf_collect()
3846 			if (sh->sh_flags & SHF_EXECINSTR) {  in bpf_object__elf_collect()
3848 					obj->efile.text_shndx = idx;  in bpf_object__elf_collect()
3854 				sec_desc->sec_type = SEC_DATA;  in bpf_object__elf_collect()
3855 				sec_desc->shdr = sh;  in bpf_object__elf_collect()
3856 				sec_desc->data = data;  in bpf_object__elf_collect()
3859 				sec_desc->sec_type = SEC_RODATA;  in bpf_object__elf_collect()
3860 				sec_desc->shdr = sh;  in bpf_object__elf_collect()
3861 				sec_desc->data = data;  in bpf_object__elf_collect()
3866 				sec_desc->sec_type = SEC_ST_OPS;  in bpf_object__elf_collect()
3867 				sec_desc->shdr = sh;  in bpf_object__elf_collect()
3868 				sec_desc->data = data;  in bpf_object__elf_collect()
3869 				obj->efile.has_st_ops = true;  in bpf_object__elf_collect()
3871 				obj->efile.arena_data = data;  in bpf_object__elf_collect()
3872 				obj->efile.arena_data_shndx = idx;  in bpf_object__elf_collect()
3877 		} else if (sh->sh_type == SHT_REL) {  in bpf_object__elf_collect()
3878 			int targ_sec_idx = sh->sh_info; /* points to other section */  in bpf_object__elf_collect()
3880 			if (sh->sh_entsize != sizeof(Elf64_Rel) ||  in bpf_object__elf_collect()
3881 			    targ_sec_idx >= obj->efile.sec_cnt)  in bpf_object__elf_collect()
3882 				return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3897 			sec_desc->sec_type = SEC_RELO;  in bpf_object__elf_collect()
3898 			sec_desc->shdr = sh;  in bpf_object__elf_collect()
3899 			sec_desc->data = data;  in bpf_object__elf_collect()
3900 		} else if (sh->sh_type == SHT_NOBITS && (strcmp(name, BSS_SEC) == 0 ||  in bpf_object__elf_collect()
3902 			sec_desc->sec_type = SEC_BSS;  in bpf_object__elf_collect()
3903 			sec_desc->shdr = sh;  in bpf_object__elf_collect()
3904 			sec_desc->data = data;  in bpf_object__elf_collect()
3907 				(size_t)sh->sh_size);  in bpf_object__elf_collect()
3911 	if (!obj->efile.strtabidx || obj->efile.strtabidx > idx) {  in bpf_object__elf_collect()
3912 		pr_warn("elf: symbol strings section missing or invalid in %s\n", obj->path);  in bpf_object__elf_collect()
3913 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__elf_collect()
3916 	/* sort BPF programs by section name and in-section instruction offset  in bpf_object__elf_collect()
3919 	if (obj->nr_programs)  in bpf_object__elf_collect()
3920 		qsort(obj->programs, obj->nr_programs, sizeof(*obj->programs), cmp_progs);  in bpf_object__elf_collect()
3927 	int bind = ELF64_ST_BIND(sym->st_info);  in sym_is_extern()
3929 	return sym->st_shndx == SHN_UNDEF &&  in sym_is_extern()
3931 	       ELF64_ST_TYPE(sym->st_info) == STT_NOTYPE;  in sym_is_extern()
3936 	int bind = ELF64_ST_BIND(sym->st_info);  in sym_is_subprog()
3937 	int type = ELF64_ST_TYPE(sym->st_info);  in sym_is_subprog()
3940 	if (sym->st_shndx != text_shndx)  in sym_is_subprog()
3958 		return -ESRCH;  in find_extern_btf_id()
3967 		tname = btf__name_by_offset(btf, t->name_off);  in find_extern_btf_id()
3972 		    btf_var(t)->linkage != BTF_VAR_GLOBAL_EXTERN)  in find_extern_btf_id()
3973 			return -EINVAL;  in find_extern_btf_id()
3976 			return -EINVAL;  in find_extern_btf_id()
3981 	return -ENOENT;  in find_extern_btf_id()
3990 		return -ESRCH;  in find_extern_sec_btf_id()
4001 			if (vs->type == ext_btf_id)  in find_extern_sec_btf_id()
4006 	return -ENOENT;  in find_extern_sec_btf_id()
4016 	name = btf__name_by_offset(btf, t->name_off);  in find_kcfg_type()
4025 			return t->size == 1 ? KCFG_BOOL : KCFG_UNKNOWN;  in find_kcfg_type()
4028 		if (t->size == 1)  in find_kcfg_type()
4030 		if (t->size < 1 || t->size > 8 || (t->size & (t->size - 1)))  in find_kcfg_type()
4035 		if (t->size != 4)  in find_kcfg_type()
4045 		if (btf_array(t)->nelems == 0)  in find_kcfg_type()
4047 		if (find_kcfg_type(btf, btf_array(t)->type, NULL) != KCFG_CHAR)  in find_kcfg_type()
4060 	if (a->type != b->type)  in cmp_externs()
4061 		return a->type < b->type ? -1 : 1;  in cmp_externs()
4063 	if (a->type == EXT_KCFG) {  in cmp_externs()
4065 		if (a->kcfg.align != b->kcfg.align)  in cmp_externs()
4066 			return a->kcfg.align > b->kcfg.align ? -1 : 1;  in cmp_externs()
4068 		if (a->kcfg.sz != b->kcfg.sz)  in cmp_externs()
4069 			return a->kcfg.sz < b->kcfg.sz ? -1 : 1;  in cmp_externs()
4073 	return strcmp(a->name, b->name);  in cmp_externs()
4111 		vt = btf__type_by_id(btf, vs->type);  in add_dummy_ksym_var()
4135 	struct extern_desc *ext;  in bpf_object__collect_externs()  local
4142 	if (!obj->efile.symbols)  in bpf_object__collect_externs()
4145 	scn = elf_sec_by_idx(obj, obj->efile.symbols_shndx);  in bpf_object__collect_externs()
4147 	if (!sh || sh->sh_entsize != sizeof(Elf64_Sym))  in bpf_object__collect_externs()
4148 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_externs()
4150 	dummy_var_btf_id = add_dummy_ksym_var(obj->btf);  in bpf_object__collect_externs()
4154 	n = sh->sh_size / sh->sh_entsize;  in bpf_object__collect_externs()
4161 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_externs()
4164 		ext_name = elf_sym_str(obj, sym->st_name);  in bpf_object__collect_externs()
4168 		ext = obj->externs;  in bpf_object__collect_externs()
4169 		ext = libbpf_reallocarray(ext, obj->nr_extern + 1, sizeof(*ext));  in bpf_object__collect_externs()
4170 		if (!ext)  in bpf_object__collect_externs()
4171 			return -ENOMEM;  in bpf_object__collect_externs()
4172 		obj->externs = ext;  in bpf_object__collect_externs()
4173 		ext = &ext[obj->nr_extern];  in bpf_object__collect_externs()
4174 		memset(ext, 0, sizeof(*ext));  in bpf_object__collect_externs()
4175 		obj->nr_extern++;  in bpf_object__collect_externs()
4177 		ext->btf_id = find_extern_btf_id(obj->btf, ext_name);  in bpf_object__collect_externs()
4178 		if (ext->btf_id <= 0) {  in bpf_object__collect_externs()
4180 				ext_name, ext->btf_id);  in bpf_object__collect_externs()
4181 			return ext->btf_id;  in bpf_object__collect_externs()
4183 		t = btf__type_by_id(obj->btf, ext->btf_id);  in bpf_object__collect_externs()
4184 		ext->name = btf__name_by_offset(obj->btf, t->name_off);  in bpf_object__collect_externs()
4185 		ext->sym_idx = i;  in bpf_object__collect_externs()
4186 		ext->is_weak = ELF64_ST_BIND(sym->st_info) == STB_WEAK;  in bpf_object__collect_externs()
4188 		ext_essent_len = bpf_core_essential_name_len(ext->name);  in bpf_object__collect_externs()
4189 		ext->essent_name = NULL;  in bpf_object__collect_externs()
4190 		if (ext_essent_len != strlen(ext->name)) {  in bpf_object__collect_externs()
4191 			ext->essent_name = strndup(ext->name, ext_essent_len);  in bpf_object__collect_externs()
4192 			if (!ext->essent_name)  in bpf_object__collect_externs()
4193 				return -ENOMEM;  in bpf_object__collect_externs()
4196 		ext->sec_btf_id = find_extern_sec_btf_id(obj->btf, ext->btf_id);  in bpf_object__collect_externs()
4197 		if (ext->sec_btf_id <= 0) {  in bpf_object__collect_externs()
4199 				ext_name, ext->btf_id, ext->sec_btf_id);  in bpf_object__collect_externs()
4200 			return ext->sec_btf_id;  in bpf_object__collect_externs()
4202 		sec = (void *)btf__type_by_id(obj->btf, ext->sec_btf_id);  in bpf_object__collect_externs()
4203 		sec_name = btf__name_by_offset(obj->btf, sec->name_off);  in bpf_object__collect_externs()
4208 					ext->name, KCONFIG_SEC);  in bpf_object__collect_externs()
4209 				return -ENOTSUP;  in bpf_object__collect_externs()
4212 			ext->type = EXT_KCFG;  in bpf_object__collect_externs()
4213 			ext->kcfg.sz = btf__resolve_size(obj->btf, t->type);  in bpf_object__collect_externs()
4214 			if (ext->kcfg.sz <= 0) {  in bpf_object__collect_externs()
4216 					ext_name, ext->kcfg.sz);  in bpf_object__collect_externs()
4217 				return ext->kcfg.sz;  in bpf_object__collect_externs()
4219 			ext->kcfg.align = btf__align_of(obj->btf, t->type);  in bpf_object__collect_externs()
4220 			if (ext->kcfg.align <= 0) {  in bpf_object__collect_externs()
4222 					ext_name, ext->kcfg.align);  in bpf_object__collect_externs()
4223 				return -EINVAL;  in bpf_object__collect_externs()
4225 			ext->kcfg.type = find_kcfg_type(obj->btf, t->type,  in bpf_object__collect_externs()
4226 							&ext->kcfg.is_signed);  in bpf_object__collect_externs()
4227 			if (ext->kcfg.type == KCFG_UNKNOWN) {  in bpf_object__collect_externs()
4229 				return -ENOTSUP;  in bpf_object__collect_externs()
4233 			ext->type = EXT_KSYM;  in bpf_object__collect_externs()
4234 			skip_mods_and_typedefs(obj->btf, t->type,  in bpf_object__collect_externs()
4235 					       &ext->ksym.type_id);  in bpf_object__collect_externs()
4238 			return -ENOTSUP;  in bpf_object__collect_externs()
4241 	pr_debug("collected %d externs total\n", obj->nr_extern);  in bpf_object__collect_externs()
4243 	if (!obj->nr_extern)  in bpf_object__collect_externs()
4247 	qsort(obj->externs, obj->nr_extern, sizeof(*ext), cmp_externs);  in bpf_object__collect_externs()
4251 	 * pretending that each extern is a 8-byte variable  in bpf_object__collect_externs()
4254 		/* find existing 4-byte integer type in BTF to use for fake  in bpf_object__collect_externs()
4257 		int int_btf_id = find_int_btf_id(obj->btf);  in bpf_object__collect_externs()
4259 		 * will be used to replace the vs->type and  in bpf_object__collect_externs()
4265 		dummy_var = btf__type_by_id(obj->btf, dummy_var_btf_id);  in bpf_object__collect_externs()
4266 		for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__collect_externs()
4267 			ext = &obj->externs[i];  in bpf_object__collect_externs()
4268 			if (ext->type != EXT_KSYM)  in bpf_object__collect_externs()
4271 				 i, ext->sym_idx, ext->name);  in bpf_object__collect_externs()
4280 			vt = (void *)btf__type_by_id(obj->btf, vs->type);  in bpf_object__collect_externs()
4281 			ext_name = btf__name_by_offset(obj->btf, vt->name_off);  in bpf_object__collect_externs()
4282 			ext = find_extern_by_name(obj, ext_name);  in bpf_object__collect_externs()
4283 			if (!ext) {  in bpf_object__collect_externs()
4286 				return -ESRCH;  in bpf_object__collect_externs()
4293 				func_proto = btf__type_by_id(obj->btf,  in bpf_object__collect_externs()
4294 							     vt->type);  in bpf_object__collect_externs()
4302 							dummy_var->name_off;  in bpf_object__collect_externs()
4303 				vs->type = dummy_var_btf_id;  in bpf_object__collect_externs()
4304 				vt->info &= ~0xffff;  in bpf_object__collect_externs()
4305 				vt->info |= BTF_FUNC_GLOBAL;  in bpf_object__collect_externs()
4307 				btf_var(vt)->linkage = BTF_VAR_GLOBAL_ALLOCATED;  in bpf_object__collect_externs()
4308 				vt->type = int_btf_id;  in bpf_object__collect_externs()
4310 			vs->offset = off;  in bpf_object__collect_externs()
4311 			vs->size = sizeof(int);  in bpf_object__collect_externs()
4313 		sec->size = off;  in bpf_object__collect_externs()
4318 		/* for kcfg externs calculate their offsets within a .kconfig map */  in bpf_object__collect_externs()
4320 		for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__collect_externs()
4321 			ext = &obj->externs[i];  in bpf_object__collect_externs()
4322 			if (ext->type != EXT_KCFG)  in bpf_object__collect_externs()
4325 			ext->kcfg.data_off = roundup(off, ext->kcfg.align);  in bpf_object__collect_externs()
4326 			off = ext->kcfg.data_off + ext->kcfg.sz;  in bpf_object__collect_externs()
4328 				 i, ext->sym_idx, ext->kcfg.data_off, ext->name);  in bpf_object__collect_externs()
4330 		sec->size = off;  in bpf_object__collect_externs()
4335 			t = btf__type_by_id(obj->btf, vs->type);  in bpf_object__collect_externs()
4336 			ext_name = btf__name_by_offset(obj->btf, t->name_off);  in bpf_object__collect_externs()
4337 			ext = find_extern_by_name(obj, ext_name);  in bpf_object__collect_externs()
4338 			if (!ext) {  in bpf_object__collect_externs()
4341 				return -ESRCH;  in bpf_object__collect_externs()
4343 			btf_var(t)->linkage = BTF_VAR_GLOBAL_ALLOCATED;  in bpf_object__collect_externs()
4344 			vs->offset = ext->kcfg.data_off;  in bpf_object__collect_externs()
4352 	return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1;  in prog_is_subprog()
4364 		if (!strcmp(prog->name, name))  in bpf_object__find_program_by_name()
4373 	switch (obj->efile.secs[shndx].sec_type) {  in bpf_object__shndx_is_data()
4386 	return shndx == obj->efile.btf_maps_shndx;  in bpf_object__shndx_is_maps()
4392 	if (shndx == obj->efile.symbols_shndx)  in bpf_object__section_to_libbpf_map_type()
4395 	switch (obj->efile.secs[shndx].sec_type) {  in bpf_object__section_to_libbpf_map_type()
4412 	struct bpf_insn *insn = &prog->insns[insn_idx];  in bpf_program__record_reloc()
4413 	size_t map_idx, nr_maps = prog->obj->nr_maps;  in bpf_program__record_reloc()
4414 	struct bpf_object *obj = prog->obj;  in bpf_program__record_reloc()
4415 	__u32 shdr_idx = sym->st_shndx;  in bpf_program__record_reloc()
4422 			prog->name, sym_name, insn_idx, insn->code);  in bpf_program__record_reloc()
4423 		return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4427 		int sym_idx = ELF64_R_SYM(rel->r_info);  in bpf_program__record_reloc()
4428 		int i, n = obj->nr_extern;  in bpf_program__record_reloc()
4429 		struct extern_desc *ext;  in bpf_program__record_reloc()  local
4432 			ext = &obj->externs[i];  in bpf_program__record_reloc()
4433 			if (ext->sym_idx == sym_idx)  in bpf_program__record_reloc()
4438 				prog->name, sym_name, sym_idx);  in bpf_program__record_reloc()
4439 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4442 			 prog->name, i, ext->name, ext->sym_idx, insn_idx);  in bpf_program__record_reloc()
4443 		if (insn->code == (BPF_JMP | BPF_CALL))  in bpf_program__record_reloc()
4444 			reloc_desc->type = RELO_EXTERN_CALL;  in bpf_program__record_reloc()
4446 			reloc_desc->type = RELO_EXTERN_LD64;  in bpf_program__record_reloc()
4447 		reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4448 		reloc_desc->ext_idx = i;  in bpf_program__record_reloc()
4452 	/* sub-program call relocation */  in bpf_program__record_reloc()
4454 		if (insn->src_reg != BPF_PSEUDO_CALL) {  in bpf_program__record_reloc()
4455 			pr_warn("prog '%s': incorrect bpf_call opcode\n", prog->name);  in bpf_program__record_reloc()
4456 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4459 		if (!shdr_idx || shdr_idx != obj->efile.text_shndx) {  in bpf_program__record_reloc()
4462 				prog->name, sym_name, sym_sec_name);  in bpf_program__record_reloc()
4463 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4465 		if (sym->st_value % BPF_INSN_SZ) {  in bpf_program__record_reloc()
4467 				prog->name, sym_name, (size_t)sym->st_value);  in bpf_program__record_reloc()
4468 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4470 		reloc_desc->type = RELO_CALL;  in bpf_program__record_reloc()
4471 		reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4472 		reloc_desc->sym_off = sym->st_value;  in bpf_program__record_reloc()
4478 			prog->name, sym_name, shdr_idx);  in bpf_program__record_reloc()
4479 		return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4483 	if (sym_is_subprog(sym, obj->efile.text_shndx)) {  in bpf_program__record_reloc()
4484 		/* global_func: sym->st_value = offset in the section, insn->imm = 0.  in bpf_program__record_reloc()
4485 		 * local_func: sym->st_value = 0, insn->imm = offset in the section.  in bpf_program__record_reloc()
4487 		if ((sym->st_value % BPF_INSN_SZ) || (insn->imm % BPF_INSN_SZ)) {  in bpf_program__record_reloc()
4489 				prog->name, sym_name, (size_t)sym->st_value, insn->imm);  in bpf_program__record_reloc()
4490 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4493 		reloc_desc->type = RELO_SUBPROG_ADDR;  in bpf_program__record_reloc()
4494 		reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4495 		reloc_desc->sym_off = sym->st_value;  in bpf_program__record_reloc()
4503 	if (shdr_idx == obj->efile.arena_data_shndx) {  in bpf_program__record_reloc()
4504 		reloc_desc->type = RELO_DATA;  in bpf_program__record_reloc()
4505 		reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4506 		reloc_desc->map_idx = obj->arena_map - obj->maps;  in bpf_program__record_reloc()
4507 		reloc_desc->sym_off = sym->st_value;  in bpf_program__record_reloc()
4515 				prog->name, sym_name, sym_sec_name);  in bpf_program__record_reloc()
4516 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4519 			map = &obj->maps[map_idx];  in bpf_program__record_reloc()
4520 			if (map->libbpf_type != type ||  in bpf_program__record_reloc()
4521 			    map->sec_idx != sym->st_shndx ||  in bpf_program__record_reloc()
4522 			    map->sec_offset != sym->st_value)  in bpf_program__record_reloc()
4525 				 prog->name, map_idx, map->name, map->sec_idx,  in bpf_program__record_reloc()
4526 				 map->sec_offset, insn_idx);  in bpf_program__record_reloc()
4531 				prog->name, sym_sec_name, (size_t)sym->st_value);  in bpf_program__record_reloc()
4532 			return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4534 		reloc_desc->type = RELO_LD64;  in bpf_program__record_reloc()
4535 		reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4536 		reloc_desc->map_idx = map_idx;  in bpf_program__record_reloc()
4537 		reloc_desc->sym_off = 0; /* sym->st_value determines map_idx */  in bpf_program__record_reloc()
4544 			prog->name, sym_sec_name);  in bpf_program__record_reloc()
4545 		return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4548 		map = &obj->maps[map_idx];  in bpf_program__record_reloc()
4549 		if (map->libbpf_type != type || map->sec_idx != sym->st_shndx)  in bpf_program__record_reloc()
4552 			 prog->name, map_idx, map->name, map->sec_idx,  in bpf_program__record_reloc()
4553 			 map->sec_offset, insn_idx);  in bpf_program__record_reloc()
4558 			prog->name, sym_sec_name);  in bpf_program__record_reloc()
4559 		return -LIBBPF_ERRNO__RELOC;  in bpf_program__record_reloc()
4562 	reloc_desc->type = RELO_DATA;  in bpf_program__record_reloc()
4563 	reloc_desc->insn_idx = insn_idx;  in bpf_program__record_reloc()
4564 	reloc_desc->map_idx = map_idx;  in bpf_program__record_reloc()
4565 	reloc_desc->sym_off = sym->st_value;  in bpf_program__record_reloc()
4571 	return insn_idx >= prog->sec_insn_off &&  in prog_contains_insn()
4572 	       insn_idx < prog->sec_insn_off + prog->sec_insn_cnt;  in prog_contains_insn()
4578 	int l = 0, r = obj->nr_programs - 1, m;  in find_prog_by_sec_insn()
4581 	if (!obj->nr_programs)  in find_prog_by_sec_insn()
4585 		m = l + (r - l + 1) / 2;  in find_prog_by_sec_insn()
4586 		prog = &obj->programs[m];  in find_prog_by_sec_insn()
4588 		if (prog->sec_idx < sec_idx ||  in find_prog_by_sec_insn()
4589 		    (prog->sec_idx == sec_idx && prog->sec_insn_off <= insn_idx))  in find_prog_by_sec_insn()
4592 			r = m - 1;  in find_prog_by_sec_insn()
4597 	prog = &obj->programs[l];  in find_prog_by_sec_insn()
4598 	if (prog->sec_idx == sec_idx && prog_contains_insn(prog, insn_idx))  in find_prog_by_sec_insn()
4607 	size_t sec_idx = shdr->sh_info, sym_idx;  in bpf_object__collect_prog_relos()
4618 	if (sec_idx >= obj->efile.sec_cnt)  in bpf_object__collect_prog_relos()
4619 		return -EINVAL;  in bpf_object__collect_prog_relos()
4624 		return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_prog_relos()
4626 	relo_sec_name = elf_sec_str(obj, shdr->sh_name);  in bpf_object__collect_prog_relos()
4629 		return -EINVAL;  in bpf_object__collect_prog_relos()
4633 	nrels = shdr->sh_size / shdr->sh_entsize;  in bpf_object__collect_prog_relos()
4639 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_prog_relos()
4642 		sym_idx = ELF64_R_SYM(rel->r_info);  in bpf_object__collect_prog_relos()
4647 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_prog_relos()
4650 		if (sym->st_shndx >= obj->efile.sec_cnt) {  in bpf_object__collect_prog_relos()
4652 				relo_sec_name, sym_idx, (size_t)sym->st_shndx, i);  in bpf_object__collect_prog_relos()
4653 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_prog_relos()
4656 		if (rel->r_offset % BPF_INSN_SZ || rel->r_offset >= scn_data->d_size) {  in bpf_object__collect_prog_relos()
4658 				relo_sec_name, (size_t)rel->r_offset, i);  in bpf_object__collect_prog_relos()
4659 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_prog_relos()
4662 		insn_idx = rel->r_offset / BPF_INSN_SZ;  in bpf_object__collect_prog_relos()
4669 		if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION && sym->st_name == 0)  in bpf_object__collect_prog_relos()
4670 			sym_name = elf_sec_name(obj, elf_sec_by_idx(obj, sym->st_shndx));  in bpf_object__collect_prog_relos()
4672 			sym_name = elf_sym_str(obj, sym->st_name);  in bpf_object__collect_prog_relos()
4685 		relos = libbpf_reallocarray(prog->reloc_desc,  in bpf_object__collect_prog_relos()
4686 					    prog->nr_reloc + 1, sizeof(*relos));  in bpf_object__collect_prog_relos()
4688 			return -ENOMEM;  in bpf_object__collect_prog_relos()
4689 		prog->reloc_desc = relos;  in bpf_object__collect_prog_relos()
4692 		insn_idx -= prog->sec_insn_off;  in bpf_object__collect_prog_relos()
4693 		err = bpf_program__record_reloc(prog, &relos[prog->nr_reloc],  in bpf_object__collect_prog_relos()
4698 		prog->nr_reloc++;  in bpf_object__collect_prog_relos()
4707 	if (!obj->btf)  in map_fill_btf_type_info()
4708 		return -ENOENT;  in map_fill_btf_type_info()
4710 	/* if it's BTF-defined map, we don't need to search for type IDs.  in map_fill_btf_type_info()
4714 	if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map))  in map_fill_btf_type_info()
4722 		return -ENOENT;  in map_fill_btf_type_info()
4724 	id = btf__find_by_name(obj->btf, map->real_name);  in map_fill_btf_type_info()
4728 	map->btf_key_type_id = 0;  in map_fill_btf_type_info()
4729 	map->btf_value_type_id = id;  in map_fill_btf_type_info()
4745 		err = -errno;  in bpf_get_map_info_from_fdinfo()
4753 			info->type = val;  in bpf_get_map_info_from_fdinfo()
4755 			info->key_size = val;  in bpf_get_map_info_from_fdinfo()
4757 			info->value_size = val;  in bpf_get_map_info_from_fdinfo()
4759 			info->max_entries = val;  in bpf_get_map_info_from_fdinfo()
4761 			info->map_flags = val;  in bpf_get_map_info_from_fdinfo()
4771 	return map->autocreate;  in bpf_map__autocreate()
4776 	if (map->obj->loaded)  in bpf_map__set_autocreate()
4777 		return libbpf_err(-EBUSY);  in bpf_map__set_autocreate()
4779 	map->autocreate = autocreate;  in bpf_map__set_autocreate()
4798 	if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0)  in bpf_map__reuse_fd()
4799 		new_name = strdup(map->name);  in bpf_map__reuse_fd()
4804 		return libbpf_err(-errno);  in bpf_map__reuse_fd()
4813 		err = -errno;  in bpf_map__reuse_fd()
4817 	err = reuse_fd(map->fd, new_fd);  in bpf_map__reuse_fd()
4821 	free(map->name);  in bpf_map__reuse_fd()
4823 	map->name = new_name;  in bpf_map__reuse_fd()
4824 	map->def.type = info.type;  in bpf_map__reuse_fd()
4825 	map->def.key_size = info.key_size;  in bpf_map__reuse_fd()
4826 	map->def.value_size = info.value_size;  in bpf_map__reuse_fd()
4827 	map->def.max_entries = info.max_entries;  in bpf_map__reuse_fd()
4828 	map->def.map_flags = info.map_flags;  in bpf_map__reuse_fd()
4829 	map->btf_key_type_id = info.btf_key_type_id;  in bpf_map__reuse_fd()
4830 	map->btf_value_type_id = info.btf_value_type_id;  in bpf_map__reuse_fd()
4831 	map->reused = true;  in bpf_map__reuse_fd()
4832 	map->map_extra = info.map_extra;  in bpf_map__reuse_fd()
4843 	return map->def.max_entries;  in bpf_map__max_entries()
4848 	if (!bpf_map_type__is_map_in_map(map->def.type))  in bpf_map__inner_map()
4851 	return map->inner_map;  in bpf_map__inner_map()
4856 	if (map->obj->loaded)  in bpf_map__set_max_entries()
4857 		return libbpf_err(-EBUSY);  in bpf_map__set_max_entries()
4859 	map->def.max_entries = max_entries;  in bpf_map__set_max_entries()
4861 	/* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */  in bpf_map__set_max_entries()
4863 		map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries);  in bpf_map__set_max_entries()
4871 	int bpffs_fd = -1, token_fd, err;  in bpf_object_prepare_token()
4876 	if (obj->token_path && obj->token_path[0] == '\0') {  in bpf_object_prepare_token()
4877 		pr_debug("object '%s': token is prevented, skipping...\n", obj->name);  in bpf_object_prepare_token()
4881 	mandatory = obj->token_path != NULL;  in bpf_object_prepare_token()
4884 	bpffs_path = obj->token_path ?: BPF_FS_DEFAULT_PATH;  in bpf_object_prepare_token()
4887 		err = -errno;  in bpf_object_prepare_token()
4889 		     obj->name, err, bpffs_path,  in bpf_object_prepare_token()
4897 		if (!mandatory && token_fd == -ENOENT) {  in bpf_object_prepare_token()
4899 				 obj->name, bpffs_path);  in bpf_object_prepare_token()
4903 		     obj->name, token_fd, bpffs_path,  in bpf_object_prepare_token()
4908 	obj->feat_cache = calloc(1, sizeof(*obj->feat_cache));  in bpf_object_prepare_token()
4909 	if (!obj->feat_cache) {  in bpf_object_prepare_token()
4911 		return -ENOMEM;  in bpf_object_prepare_token()
4914 	obj->token_fd = token_fd;  in bpf_object_prepare_token()
4915 	obj->feat_cache->token_fd = token_fd;  in bpf_object_prepare_token()
4930 		.token_fd = obj->token_fd,  in bpf_object__probe_loading()
4931 		.prog_flags = obj->token_fd ? BPF_F_TOKEN_FD : 0,  in bpf_object__probe_loading()
4934 	if (obj->gen_loader)  in bpf_object__probe_loading()
4952 		return -ret;  in bpf_object__probe_loading()
4961 	if (obj->gen_loader)  in kernel_supports()
4967 	if (obj->token_fd)  in kernel_supports()
4968 		return feat_supported(obj->feat_cache, feat_id);  in kernel_supports()
4990 	return (map_info.type == map->def.type &&  in map_is_reuse_compat()
4991 		map_info.key_size == map->def.key_size &&  in map_is_reuse_compat()
4992 		map_info.value_size == map->def.value_size &&  in map_is_reuse_compat()
4993 		map_info.max_entries == map->def.max_entries &&  in map_is_reuse_compat()
4994 		map_info.map_flags == map->def.map_flags &&  in map_is_reuse_compat()
4995 		map_info.map_extra == map->map_extra);  in map_is_reuse_compat()
5004 	pin_fd = bpf_obj_get(map->pin_path);  in bpf_object__reuse_map()
5006 		err = -errno;  in bpf_object__reuse_map()
5007 		if (err == -ENOENT) {  in bpf_object__reuse_map()
5009 				 map->pin_path);  in bpf_object__reuse_map()
5013 		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));  in bpf_object__reuse_map()
5015 			map->pin_path, cp);  in bpf_object__reuse_map()
5021 			map->pin_path);  in bpf_object__reuse_map()
5023 		return -EINVAL;  in bpf_object__reuse_map()
5031 	map->pinned = true;  in bpf_object__reuse_map()
5032 	pr_debug("reused pinned map at '%s'\n", map->pin_path);  in bpf_object__reuse_map()
5040 	enum libbpf_map_type map_type = map->libbpf_type;  in bpf_object__populate_internal_map()
5044 	if (obj->gen_loader) {  in bpf_object__populate_internal_map()
5045 		bpf_gen__map_update_elem(obj->gen_loader, map - obj->maps,  in bpf_object__populate_internal_map()
5046 					 map->mmaped, map->def.value_size);  in bpf_object__populate_internal_map()
5048 			bpf_gen__map_freeze(obj->gen_loader, map - obj->maps);  in bpf_object__populate_internal_map()
5052 	err = bpf_map_update_elem(map->fd, &zero, map->mmaped, 0);  in bpf_object__populate_internal_map()
5054 		err = -errno;  in bpf_object__populate_internal_map()
5057 			map->name, cp);  in bpf_object__populate_internal_map()
5061 	/* Freeze .rodata and .kconfig map as read-only from syscall side. */  in bpf_object__populate_internal_map()
5063 		err = bpf_map_freeze(map->fd);  in bpf_object__populate_internal_map()
5065 			err = -errno;  in bpf_object__populate_internal_map()
5067 			pr_warn("Error freezing map(%s) as read-only: %s\n",  in bpf_object__populate_internal_map()
5068 				map->name, cp);  in bpf_object__populate_internal_map()
5079 	return map->obj->loaded || map->reused;  in map_is_created()
5085 	struct bpf_map_def *def = &map->def;  in bpf_object__create_map()
5090 		map_name = map->name;  in bpf_object__create_map()
5091 	create_attr.map_ifindex = map->map_ifindex;  in bpf_object__create_map()
5092 	create_attr.map_flags = def->map_flags;  in bpf_object__create_map()
5093 	create_attr.numa_node = map->numa_node;  in bpf_object__create_map()
5094 	create_attr.map_extra = map->map_extra;  in bpf_object__create_map()
5095 	create_attr.token_fd = obj->token_fd;  in bpf_object__create_map()
5096 	if (obj->token_fd)  in bpf_object__create_map()
5100 		create_attr.btf_vmlinux_value_type_id = map->btf_vmlinux_value_type_id;  in bpf_object__create_map()
5101 		if (map->mod_btf_fd >= 0) {  in bpf_object__create_map()
5102 			create_attr.value_type_btf_obj_fd = map->mod_btf_fd;  in bpf_object__create_map()
5107 	if (obj->btf && btf__fd(obj->btf) >= 0) {  in bpf_object__create_map()
5108 		create_attr.btf_fd = btf__fd(obj->btf);  in bpf_object__create_map()
5109 		create_attr.btf_key_type_id = map->btf_key_type_id;  in bpf_object__create_map()
5110 		create_attr.btf_value_type_id = map->btf_value_type_id;  in bpf_object__create_map()
5113 	if (bpf_map_type__is_map_in_map(def->type)) {  in bpf_object__create_map()
5114 		if (map->inner_map) {  in bpf_object__create_map()
5115 			err = map_set_def_max_entries(map->inner_map);  in bpf_object__create_map()
5118 			err = bpf_object__create_map(obj, map->inner_map, true);  in bpf_object__create_map()
5121 					map->name, err);  in bpf_object__create_map()
5124 			map->inner_map_fd = map->inner_map->fd;  in bpf_object__create_map()
5126 		if (map->inner_map_fd >= 0)  in bpf_object__create_map()
5127 			create_attr.inner_map_fd = map->inner_map_fd;  in bpf_object__create_map()
5130 	switch (def->type) {  in bpf_object__create_map()
5148 		map->btf_key_type_id = 0;  in bpf_object__create_map()
5149 		map->btf_value_type_id = 0;  in bpf_object__create_map()
5158 	if (obj->gen_loader) {  in bpf_object__create_map()
5159 		bpf_gen__map_create(obj->gen_loader, def->type, map_name,  in bpf_object__create_map()
5160 				    def->key_size, def->value_size, def->max_entries,  in bpf_object__create_map()
5161 				    &create_attr, is_inner ? -1 : map - obj->maps);  in bpf_object__create_map()
5166 		 * will be reset to -1 eventually.  in bpf_object__create_map()
5168 		map_fd = map->fd;  in bpf_object__create_map()
5170 		map_fd = bpf_map_create(def->type, map_name,  in bpf_object__create_map()
5171 					def->key_size, def->value_size,  in bpf_object__create_map()
5172 					def->max_entries, &create_attr);  in bpf_object__create_map()
5177 		err = -errno;  in bpf_object__create_map()
5180 			map->name, cp, err);  in bpf_object__create_map()
5184 		map->btf_key_type_id = 0;  in bpf_object__create_map()
5185 		map->btf_value_type_id = 0;  in bpf_object__create_map()
5186 		map_fd = bpf_map_create(def->type, map_name,  in bpf_object__create_map()
5187 					def->key_size, def->value_size,  in bpf_object__create_map()
5188 					def->max_entries, &create_attr);  in bpf_object__create_map()
5191 	if (bpf_map_type__is_map_in_map(def->type) && map->inner_map) {  in bpf_object__create_map()
5192 		if (obj->gen_loader)  in bpf_object__create_map()
5193 			map->inner_map->fd = -1;  in bpf_object__create_map()
5194 		bpf_map__destroy(map->inner_map);  in bpf_object__create_map()
5195 		zfree(&map->inner_map);  in bpf_object__create_map()
5201 	/* obj->gen_loader case, prevent reuse_fd() from closing map_fd */  in bpf_object__create_map()
5202 	if (map->fd == map_fd)  in bpf_object__create_map()
5208 	 * map->fd stays valid but now point to what map_fd points to.  in bpf_object__create_map()
5210 	return reuse_fd(map->fd, map_fd);  in bpf_object__create_map()
5219 	for (i = 0; i < map->init_slots_sz; i++) {  in init_map_in_map_slots()
5220 		if (!map->init_slots[i])  in init_map_in_map_slots()
5223 		targ_map = map->init_slots[i];  in init_map_in_map_slots()
5224 		fd = targ_map->fd;  in init_map_in_map_slots()
5226 		if (obj->gen_loader) {  in init_map_in_map_slots()
5227 			bpf_gen__populate_outer_map(obj->gen_loader,  in init_map_in_map_slots()
5228 						    map - obj->maps, i,  in init_map_in_map_slots()
5229 						    targ_map - obj->maps);  in init_map_in_map_slots()
5231 			err = bpf_map_update_elem(map->fd, &i, &fd, 0);  in init_map_in_map_slots()
5234 			err = -errno;  in init_map_in_map_slots()
5236 				map->name, i, targ_map->name, fd, err);  in init_map_in_map_slots()
5240 			 map->name, i, targ_map->name, fd);  in init_map_in_map_slots()
5243 	zfree(&map->init_slots);  in init_map_in_map_slots()
5244 	map->init_slots_sz = 0;  in init_map_in_map_slots()
5255 	if (obj->gen_loader)  in init_prog_array_slots()
5256 		return -ENOTSUP;  in init_prog_array_slots()
5258 	for (i = 0; i < map->init_slots_sz; i++) {  in init_prog_array_slots()
5259 		if (!map->init_slots[i])  in init_prog_array_slots()
5262 		targ_prog = map->init_slots[i];  in init_prog_array_slots()
5265 		err = bpf_map_update_elem(map->fd, &i, &fd, 0);  in init_prog_array_slots()
5267 			err = -errno;  in init_prog_array_slots()
5269 				map->name, i, targ_prog->name, fd, err);  in init_prog_array_slots()
5273 			 map->name, i, targ_prog->name, fd);  in init_prog_array_slots()
5276 	zfree(&map->init_slots);  in init_prog_array_slots()
5277 	map->init_slots_sz = 0;  in init_prog_array_slots()
5287 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object_init_prog_arrays()
5288 		map = &obj->maps[i];  in bpf_object_init_prog_arrays()
5290 		if (!map->init_slots_sz || map->def.type != BPF_MAP_TYPE_PROG_ARRAY)  in bpf_object_init_prog_arrays()
5302 	if (map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY && !map->def.max_entries) {  in map_set_def_max_entries()
5308 				map->name, nr_cpus);  in map_set_def_max_entries()
5311 		pr_debug("map '%s': setting size to %d\n", map->name, nr_cpus);  in map_set_def_max_entries()
5312 		map->def.max_entries = nr_cpus;  in map_set_def_max_entries()
5327 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object__create_maps()
5328 		map = &obj->maps[i];  in bpf_object__create_maps()
5331 		 * (.rodata, .data, .kconfig, etc); later on, during program  in bpf_object__create_maps()
5332 		 * loading, if we detect that at least one of the to-be-loaded  in bpf_object__create_maps()
5337 		 * but also it allows to have CO-RE applications that use  in bpf_object__create_maps()
5339 		 * If those global variable-using programs are not loaded at  in bpf_object__create_maps()
5345 			map->autocreate = false;  in bpf_object__create_maps()
5347 		if (!map->autocreate) {  in bpf_object__create_maps()
5348 			pr_debug("map '%s': skipped auto-creating...\n", map->name);  in bpf_object__create_maps()
5358 		if (map->pin_path) {  in bpf_object__create_maps()
5362 					map->name);  in bpf_object__create_maps()
5365 			if (retried && map->fd < 0) {  in bpf_object__create_maps()
5367 					map->name);  in bpf_object__create_maps()
5368 				err = -ENOENT;  in bpf_object__create_maps()
5373 		if (map->reused) {  in bpf_object__create_maps()
5375 				 map->name, map->fd);  in bpf_object__create_maps()
5382 				 map->name, map->fd);  in bpf_object__create_maps()
5389 			if (map->def.type == BPF_MAP_TYPE_ARENA) {  in bpf_object__create_maps()
5390 				map->mmaped = mmap((void *)(long)map->map_extra,  in bpf_object__create_maps()
5392 						   map->map_extra ? MAP_SHARED | MAP_FIXED : MAP_SHARED,  in bpf_object__create_maps()
5393 						   map->fd, 0);  in bpf_object__create_maps()
5394 				if (map->mmaped == MAP_FAILED) {  in bpf_object__create_maps()
5395 					err = -errno;  in bpf_object__create_maps()
5396 					map->mmaped = NULL;  in bpf_object__create_maps()
5398 						map->name, err);  in bpf_object__create_maps()
5401 				if (obj->arena_data) {  in bpf_object__create_maps()
5402 					memcpy(map->mmaped, obj->arena_data, obj->arena_data_sz);  in bpf_object__create_maps()
5403 					zfree(&obj->arena_data);  in bpf_object__create_maps()
5406 			if (map->init_slots_sz && map->def.type != BPF_MAP_TYPE_PROG_ARRAY) {  in bpf_object__create_maps()
5413 		if (map->pin_path && !map->pinned) {  in bpf_object__create_maps()
5416 				if (!retried && err == -EEXIST) {  in bpf_object__create_maps()
5420 				pr_warn("map '%s': failed to auto-pin at '%s': %d\n",  in bpf_object__create_maps()
5421 					map->name, map->pin_path, err);  in bpf_object__create_maps()
5431 	pr_warn("map '%s': failed to create: %s(%d)\n", map->name, cp, err);  in bpf_object__create_maps()
5434 		zclose(obj->maps[j].fd);  in bpf_object__create_maps()
5448  * underscore is ignored by BPF CO-RE relocation during relocation matching.
5455 	for (i = n - 5; i >= 0; i--) {  in bpf_core_essential_name_len()
5467 	free(cands->cands);  in bpf_core_free_cands()
5484 	local_t = btf__type_by_id(local_cand->btf, local_cand->id);  in bpf_core_add_cands()
5485 	local_name = btf__str_by_offset(local_cand->btf, local_t->name_off);  in bpf_core_add_cands()
5493 		targ_name = btf__name_by_offset(targ_btf, t->name_off);  in bpf_core_add_cands()
5504 		pr_debug("CO-RE relocating [%d] %s %s: found target candidate [%d] %s %s in [%s]\n",  in bpf_core_add_cands()
5505 			 local_cand->id, btf_kind_str(local_t),  in bpf_core_add_cands()
5508 		new_cands = libbpf_reallocarray(cands->cands, cands->len + 1,  in bpf_core_add_cands()
5509 					      sizeof(*cands->cands));  in bpf_core_add_cands()
5511 			return -ENOMEM;  in bpf_core_add_cands()
5513 		cand = &new_cands[cands->len];  in bpf_core_add_cands()
5514 		cand->btf = targ_btf;  in bpf_core_add_cands()
5515 		cand->id = i;  in bpf_core_add_cands()
5517 		cands->cands = new_cands;  in bpf_core_add_cands()
5518 		cands->len++;  in bpf_core_add_cands()
5532 	if (obj->btf_modules_loaded)  in load_module_btfs()
5535 	if (obj->gen_loader)  in load_module_btfs()
5539 	obj->btf_modules_loaded = true;  in load_module_btfs()
5554 			err = -errno;  in load_module_btfs()
5563 			err = -errno;  in load_module_btfs()
5575 			err = -errno;  in load_module_btfs()
5580 		/* ignore non-module BTFs */  in load_module_btfs()
5586 		btf = btf_get_from_fd(fd, obj->btf_vmlinux);  in load_module_btfs()
5594 		err = libbpf_ensure_mem((void **)&obj->btf_modules, &obj->btf_module_cap,  in load_module_btfs()
5595 					sizeof(*obj->btf_modules), obj->btf_module_cnt + 1);  in load_module_btfs()
5599 		mod_btf = &obj->btf_modules[obj->btf_module_cnt++];  in load_module_btfs()
5601 		mod_btf->btf = btf;  in load_module_btfs()
5602 		mod_btf->id = id;  in load_module_btfs()
5603 		mod_btf->fd = fd;  in load_module_btfs()
5604 		mod_btf->name = strdup(name);  in load_module_btfs()
5605 		if (!mod_btf->name) {  in load_module_btfs()
5606 			err = -ENOMEM;  in load_module_btfs()
5634 		return ERR_PTR(-EINVAL);  in bpf_core_find_cands()
5636 	local_name = btf__name_by_offset(local_btf, local_t->name_off);  in bpf_core_find_cands()
5638 		return ERR_PTR(-EINVAL);  in bpf_core_find_cands()
5643 		return ERR_PTR(-ENOMEM);  in bpf_core_find_cands()
5646 	main_btf = obj->btf_vmlinux_override ?: obj->btf_vmlinux;  in bpf_core_find_cands()
5652 	if (cands->len)  in bpf_core_find_cands()
5656 	if (obj->btf_vmlinux_override)  in bpf_core_find_cands()
5664 	for (i = 0; i < obj->btf_module_cnt; i++) {  in bpf_core_find_cands()
5666 					 obj->btf_modules[i].btf,  in bpf_core_find_cands()
5667 					 obj->btf_modules[i].name,  in bpf_core_find_cands()
5668 					 btf__type_cnt(obj->btf_vmlinux),  in bpf_core_find_cands()
5681  * type-based CO-RE relocations and follow slightly different rules than
5682  * field-based relocations. This function assumes that root types were already
5683  * checked for name match. Beyond that initial root-level name check, names
5685  *   - any two STRUCTs/UNIONs/FWDs/ENUMs/INTs are considered compatible, but
5688  *   - for ENUMs, the size is ignored;
5689  *   - for INT, size and signedness are ignored;
5690  *   - for ARRAY, dimensionality is ignored, element types are checked for
5692  *   - CONST/VOLATILE/RESTRICT modifiers are ignored;
5693  *   - TYPEDEFs/PTRs are compatible if types they pointing to are compatible;
5694  *   - FUNC_PROTOs are compatible if they have compatible signature: same
5697  * more experience with using BPF CO-RE relocations.
5726 	relos = libbpf_reallocarray(prog->reloc_desc,  in record_relo_core()
5727 				    prog->nr_reloc + 1, sizeof(*relos));  in record_relo_core()
5729 		return -ENOMEM;  in record_relo_core()
5730 	relo = &relos[prog->nr_reloc];  in record_relo_core()
5731 	relo->type = RELO_CORE;  in record_relo_core()
5732 	relo->insn_idx = insn_idx;  in record_relo_core()
5733 	relo->core_relo = core_relo;  in record_relo_core()
5734 	prog->reloc_desc = relos;  in record_relo_core()
5735 	prog->nr_reloc++;  in record_relo_core()
5744 	for (i = 0; i < prog->nr_reloc; i++) {  in find_relo_core()
5745 		relo = &prog->reloc_desc[i];  in find_relo_core()
5746 		if (relo->type != RELO_CORE || relo->insn_idx != insn_idx)  in find_relo_core()
5749 		return relo->core_relo;  in find_relo_core()
5764 	const char *prog_name = prog->name;  in bpf_core_resolve_relo()
5767 	__u32 local_id = relo->type_id;  in bpf_core_resolve_relo()
5772 		return -EINVAL;  in bpf_core_resolve_relo()
5774 	local_name = btf__name_by_offset(local_btf, local_type->name_off);  in bpf_core_resolve_relo()
5776 		return -EINVAL;  in bpf_core_resolve_relo()
5778 	if (relo->kind != BPF_CORE_TYPE_ID_LOCAL &&  in bpf_core_resolve_relo()
5780 		cands = bpf_core_find_cands(prog->obj, local_btf, local_id);  in bpf_core_resolve_relo()
5812 	if (obj->btf_ext->core_relo_info.len == 0)  in bpf_object__relocate_core()
5816 		obj->btf_vmlinux_override = btf__parse(targ_btf_path, NULL);  in bpf_object__relocate_core()
5817 		err = libbpf_get_error(obj->btf_vmlinux_override);  in bpf_object__relocate_core()
5830 	seg = &obj->btf_ext->core_relo_info;  in bpf_object__relocate_core()
5833 		sec_idx = seg->sec_idxs[sec_num];  in bpf_object__relocate_core()
5836 		sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off);  in bpf_object__relocate_core()
5838 			err = -EINVAL;  in bpf_object__relocate_core()
5842 		pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info);  in bpf_object__relocate_core()
5845 			if (rec->insn_off % BPF_INSN_SZ)  in bpf_object__relocate_core()
5846 				return -EINVAL;  in bpf_object__relocate_core()
5847 			insn_idx = rec->insn_off / BPF_INSN_SZ;  in bpf_object__relocate_core()
5852 				 * appends all the .BTF.ext info that used to belong to that  in bpf_object__relocate_core()
5854 				 * This is similar to what x86-64 linker does for relocations.  in bpf_object__relocate_core()
5858 …pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subpro…  in bpf_object__relocate_core()
5862 			/* no need to apply CO-RE relocation if the program is  in bpf_object__relocate_core()
5865 			if (!prog->autoload)  in bpf_object__relocate_core()
5869 			 * program's frame of reference; (sub-)program code is not yet  in bpf_object__relocate_core()
5870 			 * relocated, so it's enough to just subtract in-section offset  in bpf_object__relocate_core()
5872 			insn_idx = insn_idx - prog->sec_insn_off;  in bpf_object__relocate_core()
5873 			if (insn_idx >= prog->insns_cnt)  in bpf_object__relocate_core()
5874 				return -EINVAL;  in bpf_object__relocate_core()
5875 			insn = &prog->insns[insn_idx];  in bpf_object__relocate_core()
5880 					prog->name, i, err);  in bpf_object__relocate_core()
5884 			if (prog->obj->gen_loader)  in bpf_object__relocate_core()
5887 			err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res);  in bpf_object__relocate_core()
5890 					prog->name, i, err);  in bpf_object__relocate_core()
5894 			err = bpf_core_patch_insn(prog->name, insn, insn_idx, rec, i, &targ_res);  in bpf_object__relocate_core()
5897 					prog->name, i, insn_idx, err);  in bpf_object__relocate_core()
5904 	/* obj->btf_vmlinux and module BTFs are freed after object load */  in bpf_object__relocate_core()
5905 	btf__free(obj->btf_vmlinux_override);  in bpf_object__relocate_core()
5906 	obj->btf_vmlinux_override = NULL;  in bpf_object__relocate_core()
5910 			bpf_core_free_cands(entry->pvalue);  in bpf_object__relocate_core()
5928 		 prog->name, relo_idx, insn_idx, map_idx, map->name);  in poison_map_ldimm64()
5932 		insn->code = BPF_JMP | BPF_CALL;  in poison_map_ldimm64()
5933 		insn->dst_reg = 0;  in poison_map_ldimm64()
5934 		insn->src_reg = 0;  in poison_map_ldimm64()
5935 		insn->off = 0;  in poison_map_ldimm64()
5939 		 * where lower 123 is map index into obj->maps[] array  in poison_map_ldimm64()
5941 		insn->imm = POISON_LDIMM64_MAP_BASE + map_idx;  in poison_map_ldimm64()
5953 			      int ext_idx, const struct extern_desc *ext)  in poison_kfunc_call()  argument
5956 		 prog->name, relo_idx, insn_idx, ext->name);  in poison_kfunc_call()
5959 	insn->code = BPF_JMP | BPF_CALL;  in poison_kfunc_call()
5960 	insn->dst_reg = 0;  in poison_kfunc_call()
5961 	insn->src_reg = 0;  in poison_kfunc_call()
5962 	insn->off = 0;  in poison_kfunc_call()
5966 	 * where lower 123 is extern index into obj->externs[] array  in poison_kfunc_call()
5968 	insn->imm = POISON_CALL_KFUNC_BASE + ext_idx;  in poison_kfunc_call()
5972  *  - map references;
5973  *  - global variable references;
5974  *  - extern references.
5981 	for (i = 0; i < prog->nr_reloc; i++) {  in bpf_object__relocate_data()
5982 		struct reloc_desc *relo = &prog->reloc_desc[i];  in bpf_object__relocate_data()
5983 		struct bpf_insn *insn = &prog->insns[relo->insn_idx];  in bpf_object__relocate_data()
5985 		struct extern_desc *ext;  in bpf_object__relocate_data()  local
5987 		switch (relo->type) {  in bpf_object__relocate_data()
5989 			map = &obj->maps[relo->map_idx];  in bpf_object__relocate_data()
5990 			if (obj->gen_loader) {  in bpf_object__relocate_data()
5992 				insn[0].imm = relo->map_idx;  in bpf_object__relocate_data()
5993 			} else if (map->autocreate) {  in bpf_object__relocate_data()
5995 				insn[0].imm = map->fd;  in bpf_object__relocate_data()
5997 				poison_map_ldimm64(prog, i, relo->insn_idx, insn,  in bpf_object__relocate_data()
5998 						   relo->map_idx, map);  in bpf_object__relocate_data()
6002 			map = &obj->maps[relo->map_idx];  in bpf_object__relocate_data()
6003 			insn[1].imm = insn[0].imm + relo->sym_off;  in bpf_object__relocate_data()
6004 			if (obj->gen_loader) {  in bpf_object__relocate_data()
6006 				insn[0].imm = relo->map_idx;  in bpf_object__relocate_data()
6007 			} else if (map->autocreate) {  in bpf_object__relocate_data()
6009 				insn[0].imm = map->fd;  in bpf_object__relocate_data()
6011 				poison_map_ldimm64(prog, i, relo->insn_idx, insn,  in bpf_object__relocate_data()
6012 						   relo->map_idx, map);  in bpf_object__relocate_data()
6016 			ext = &obj->externs[relo->ext_idx];  in bpf_object__relocate_data()
6017 			if (ext->type == EXT_KCFG) {  in bpf_object__relocate_data()
6018 				if (obj->gen_loader) {  in bpf_object__relocate_data()
6020 					insn[0].imm = obj->kconfig_map_idx;  in bpf_object__relocate_data()
6023 					insn[0].imm = obj->maps[obj->kconfig_map_idx].fd;  in bpf_object__relocate_data()
6025 				insn[1].imm = ext->kcfg.data_off;  in bpf_object__relocate_data()
6027 				if (ext->ksym.type_id && ext->is_set) { /* typed ksyms */  in bpf_object__relocate_data()
6029 					insn[0].imm = ext->ksym.kernel_btf_id;  in bpf_object__relocate_data()
6030 					insn[1].imm = ext->ksym.kernel_btf_obj_fd;  in bpf_object__relocate_data()
6032 					insn[0].imm = (__u32)ext->ksym.addr;  in bpf_object__relocate_data()
6033 					insn[1].imm = ext->ksym.addr >> 32;  in bpf_object__relocate_data()
6038 			ext = &obj->externs[relo->ext_idx];  in bpf_object__relocate_data()
6040 			if (ext->is_set) {  in bpf_object__relocate_data()
6041 				insn[0].imm = ext->ksym.kernel_btf_id;  in bpf_object__relocate_data()
6042 				insn[0].off = ext->ksym.btf_fd_idx;  in bpf_object__relocate_data()
6044 				poison_kfunc_call(prog, i, relo->insn_idx, insn,  in bpf_object__relocate_data()
6045 						  relo->ext_idx, ext);  in bpf_object__relocate_data()
6051 					prog->name, i);  in bpf_object__relocate_data()
6052 				return -EINVAL;  in bpf_object__relocate_data()
6064 				prog->name, i, relo->type);  in bpf_object__relocate_data()
6065 			return -EINVAL;  in bpf_object__relocate_data()
6086 		sec_idx = ext_info->sec_idxs[sec_num];  in adjust_prog_btf_ext_info()
6088 		if (prog->sec_idx != sec_idx)  in adjust_prog_btf_ext_info()
6094 			if (insn_off < prog->sec_insn_off)  in adjust_prog_btf_ext_info()
6096 			if (insn_off >= prog->sec_insn_off + prog->sec_insn_cnt)  in adjust_prog_btf_ext_info()
6101 			copy_end = rec + ext_info->rec_size;  in adjust_prog_btf_ext_info()
6105 			return -ENOENT;  in adjust_prog_btf_ext_info()
6107 		/* append func/line info of a given (sub-)program to the main  in adjust_prog_btf_ext_info()
6110 		old_sz = (size_t)(*prog_rec_cnt) * ext_info->rec_size;  in adjust_prog_btf_ext_info()
6111 		new_sz = old_sz + (copy_end - copy_start);  in adjust_prog_btf_ext_info()
6114 			return -ENOMEM;  in adjust_prog_btf_ext_info()
6116 		*prog_rec_cnt = new_sz / ext_info->rec_size;  in adjust_prog_btf_ext_info()
6117 		memcpy(new_prog_info + old_sz, copy_start, copy_end - copy_start);  in adjust_prog_btf_ext_info()
6119 		/* Kernel instruction offsets are in units of 8-byte  in adjust_prog_btf_ext_info()
6120 		 * instructions, while .BTF.ext instruction offsets generated  in adjust_prog_btf_ext_info()
6125 		off_adj = prog->sub_insn_off - prog->sec_insn_off;  in adjust_prog_btf_ext_info()
6128 		for (; rec < rec_end; rec += ext_info->rec_size) {  in adjust_prog_btf_ext_info()
6133 		*prog_rec_sz = ext_info->rec_size;  in adjust_prog_btf_ext_info()
6137 	return -ENOENT;  in adjust_prog_btf_ext_info()
6147 	/* no .BTF.ext relocation if .BTF.ext is missing or kernel doesn't  in reloc_prog_func_and_line_info()
6150 	if (!obj->btf_ext || !kernel_supports(obj, FEAT_BTF_FUNC))  in reloc_prog_func_and_line_info()
6156 	if (main_prog != prog && !main_prog->func_info)  in reloc_prog_func_and_line_info()
6159 	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->func_info,  in reloc_prog_func_and_line_info()
6160 				       &main_prog->func_info,  in reloc_prog_func_and_line_info()
6161 				       &main_prog->func_info_cnt,  in reloc_prog_func_and_line_info()
6162 				       &main_prog->func_info_rec_size);  in reloc_prog_func_and_line_info()
6164 		if (err != -ENOENT) {  in reloc_prog_func_and_line_info()
6165 			pr_warn("prog '%s': error relocating .BTF.ext function info: %d\n",  in reloc_prog_func_and_line_info()
6166 				prog->name, err);  in reloc_prog_func_and_line_info()
6169 		if (main_prog->func_info) {  in reloc_prog_func_and_line_info()
6174 			pr_warn("prog '%s': missing .BTF.ext function info.\n", prog->name);  in reloc_prog_func_and_line_info()
6178 …pr_warn("prog '%s': missing .BTF.ext function info for the main program, skipping all of .BTF.ext …  in reloc_prog_func_and_line_info()
6179 			prog->name);  in reloc_prog_func_and_line_info()
6184 	if (main_prog != prog && !main_prog->line_info)  in reloc_prog_func_and_line_info()
6187 	err = adjust_prog_btf_ext_info(obj, prog, &obj->btf_ext->line_info,  in reloc_prog_func_and_line_info()
6188 				       &main_prog->line_info,  in reloc_prog_func_and_line_info()
6189 				       &main_prog->line_info_cnt,  in reloc_prog_func_and_line_info()
6190 				       &main_prog->line_info_rec_size);  in reloc_prog_func_and_line_info()
6192 		if (err != -ENOENT) {  in reloc_prog_func_and_line_info()
6193 			pr_warn("prog '%s': error relocating .BTF.ext line info: %d\n",  in reloc_prog_func_and_line_info()
6194 				prog->name, err);  in reloc_prog_func_and_line_info()
6197 		if (main_prog->line_info) {  in reloc_prog_func_and_line_info()
6202 			pr_warn("prog '%s': missing .BTF.ext line info.\n", prog->name);  in reloc_prog_func_and_line_info()
6206 …pr_warn("prog '%s': missing .BTF.ext line info for the main program, skipping all of .BTF.ext line…  in reloc_prog_func_and_line_info()
6207 			prog->name);  in reloc_prog_func_and_line_info()
6217 	if (insn_idx == relo->insn_idx)  in cmp_relo_by_insn_idx()
6219 	return insn_idx < relo->insn_idx ? -1 : 1;  in cmp_relo_by_insn_idx()
6224 	if (!prog->nr_reloc)  in find_prog_insn_relo()
6226 	return bsearch(&insn_idx, prog->reloc_desc, prog->nr_reloc,  in find_prog_insn_relo()
6227 		       sizeof(*prog->reloc_desc), cmp_relo_by_insn_idx);  in find_prog_insn_relo()
6232 	int new_cnt = main_prog->nr_reloc + subprog->nr_reloc;  in append_subprog_relos()
6238 	relos = libbpf_reallocarray(main_prog->reloc_desc, new_cnt, sizeof(*relos));  in append_subprog_relos()
6244 		return -ENOMEM;  in append_subprog_relos()
6245 	if (subprog->nr_reloc)  in append_subprog_relos()
6246 		memcpy(relos + main_prog->nr_reloc, subprog->reloc_desc,  in append_subprog_relos()
6247 		       sizeof(*relos) * subprog->nr_reloc);  in append_subprog_relos()
6249 	for (i = main_prog->nr_reloc; i < new_cnt; i++)  in append_subprog_relos()
6250 		relos[i].insn_idx += subprog->sub_insn_off;  in append_subprog_relos()
6254 	main_prog->reloc_desc = relos;  in append_subprog_relos()
6255 	main_prog->nr_reloc = new_cnt;  in append_subprog_relos()
6267        subprog->sub_insn_off = main_prog->insns_cnt;  in bpf_object__append_subprog_code()
6269        new_cnt = main_prog->insns_cnt + subprog->insns_cnt;  in bpf_object__append_subprog_code()
6270        insns = libbpf_reallocarray(main_prog->insns, new_cnt, sizeof(*insns));  in bpf_object__append_subprog_code()
6272                pr_warn("prog '%s': failed to realloc prog code\n", main_prog->name);  in bpf_object__append_subprog_code()
6273                return -ENOMEM;  in bpf_object__append_subprog_code()
6275        main_prog->insns = insns;  in bpf_object__append_subprog_code()
6276        main_prog->insns_cnt = new_cnt;  in bpf_object__append_subprog_code()
6278        memcpy(main_prog->insns + subprog->sub_insn_off, subprog->insns,  in bpf_object__append_subprog_code()
6279               subprog->insns_cnt * sizeof(*insns));  in bpf_object__append_subprog_code()
6281        pr_debug("prog '%s': added %zu insns from sub-prog '%s'\n",  in bpf_object__append_subprog_code()
6282                 main_prog->name, subprog->insns_cnt, subprog->name);  in bpf_object__append_subprog_code()
6305 	for (insn_idx = 0; insn_idx < prog->sec_insn_cnt; insn_idx++) {  in bpf_object__reloc_code()
6306 		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];  in bpf_object__reloc_code()
6311 		if (relo && relo->type == RELO_EXTERN_CALL)  in bpf_object__reloc_code()
6316 		if (relo && relo->type != RELO_CALL && relo->type != RELO_SUBPROG_ADDR) {  in bpf_object__reloc_code()
6318 				prog->name, insn_idx, relo->type);  in bpf_object__reloc_code()
6319 			return -LIBBPF_ERRNO__RELOC;  in bpf_object__reloc_code()
6322 			/* sub-program instruction index is a combination of  in bpf_object__reloc_code()
6325 			 * call always has imm = -1, but for static functions  in bpf_object__reloc_code()
6326 			 * relocation is against STT_SECTION and insn->imm  in bpf_object__reloc_code()
6329 			 * for subprog addr relocation, the relo->sym_off + insn->imm is  in bpf_object__reloc_code()
6332 			if (relo->type == RELO_CALL)  in bpf_object__reloc_code()
6333 				sub_insn_idx = relo->sym_off / BPF_INSN_SZ + insn->imm + 1;  in bpf_object__reloc_code()
6335 				sub_insn_idx = (relo->sym_off + insn->imm) / BPF_INSN_SZ;  in bpf_object__reloc_code()
6342 				prog->name, insn_idx);  in bpf_object__reloc_code()
6343 			return -LIBBPF_ERRNO__RELOC;  in bpf_object__reloc_code()
6348 			 * offset necessary, insns->imm is relative to  in bpf_object__reloc_code()
6351 			sub_insn_idx = prog->sec_insn_off + insn_idx + insn->imm + 1;  in bpf_object__reloc_code()
6354 		/* we enforce that sub-programs should be in .text section */  in bpf_object__reloc_code()
6355 		subprog = find_prog_by_sec_insn(obj, obj->efile.text_shndx, sub_insn_idx);  in bpf_object__reloc_code()
6357 			pr_warn("prog '%s': no .text section found yet sub-program call exists\n",  in bpf_object__reloc_code()
6358 				prog->name);  in bpf_object__reloc_code()
6359 			return -LIBBPF_ERRNO__RELOC;  in bpf_object__reloc_code()
6365 		 *   - append it at the end of main program's instructions blog;  in bpf_object__reloc_code()
6366 		 *   - process is recursively, while current program is put on hold;  in bpf_object__reloc_code()
6367 		 *   - if that subprogram calls some other not yet processes  in bpf_object__reloc_code()
6372 		if (subprog->sub_insn_off == 0) {  in bpf_object__reloc_code()
6381 		/* main_prog->insns memory could have been re-allocated, so  in bpf_object__reloc_code()
6384 		insn = &main_prog->insns[prog->sub_insn_off + insn_idx];  in bpf_object__reloc_code()
6391 		insn->imm = subprog->sub_insn_off - (prog->sub_insn_off + insn_idx) - 1;  in bpf_object__reloc_code()
6394 			 prog->name, insn_idx, insn->imm, subprog->name, subprog->sub_insn_off);  in bpf_object__reloc_code()
6401  * Relocate sub-program calls.
6403  * Algorithm operates as follows. Each entry-point BPF program (referred to as
6404  * main prog) is processed separately. For each subprog (non-entry functions,
6413  * is into a subprog that hasn't been processed (i.e., subprog->sub_insn_off
6429  * subprog->sub_insn_off as zero at all times and won't be appended to current
6438  *        +--------+ +-------+
6440  *     +--+---+ +--+-+-+ +---+--+
6442  *     +--+---+ +------+ +---+--+
6445  *    +---+-------+   +------+----+
6447  *    +-----------+   +-----------+
6452  *    +-----------+------+
6454  *    +-----------+------+
6459  *    +-----------+------+------+
6461  *    +-----------+------+------+
6470  *    +-----------+------+
6472  *    +-----------+------+
6475  *    +-----------+------+------+
6477  *    +-----------+------+------+
6490 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__relocate_calls()
6491 		subprog = &obj->programs[i];  in bpf_object__relocate_calls()
6495 		subprog->sub_insn_off = 0;  in bpf_object__relocate_calls()
6512 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__free_relocs()
6513 		prog = &obj->programs[i];  in bpf_object__free_relocs()
6514 		zfree(&prog->reloc_desc);  in bpf_object__free_relocs()
6515 		prog->nr_reloc = 0;  in bpf_object__free_relocs()
6524 	if (a->insn_idx != b->insn_idx)  in cmp_relocs()
6525 		return a->insn_idx < b->insn_idx ? -1 : 1;  in cmp_relocs()
6528 	if (a->type != b->type)  in cmp_relocs()
6529 		return a->type < b->type ? -1 : 1;  in cmp_relocs()
6538 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__sort_relos()
6539 		struct bpf_program *p = &obj->programs[i];  in bpf_object__sort_relos()
6541 		if (!p->nr_reloc)  in bpf_object__sort_relos()
6544 		qsort(p->reloc_desc, p->nr_reloc, sizeof(*p->reloc_desc), cmp_relocs);  in bpf_object__sort_relos()
6554 	if (!obj->btf || !kernel_supports(obj, FEAT_BTF_DECL_TAG))  in bpf_prog_assign_exc_cb()
6557 	n = btf__type_cnt(obj->btf);  in bpf_prog_assign_exc_cb()
6562 		t = btf_type_by_id(obj->btf, i);  in bpf_prog_assign_exc_cb()
6563 		if (!btf_is_decl_tag(t) || btf_decl_tag(t)->component_idx != -1)  in bpf_prog_assign_exc_cb()
6566 		name = btf__str_by_offset(obj->btf, t->name_off);  in bpf_prog_assign_exc_cb()
6570 		t = btf_type_by_id(obj->btf, t->type);  in bpf_prog_assign_exc_cb()
6573 				prog->name);  in bpf_prog_assign_exc_cb()
6574 			return -EINVAL;  in bpf_prog_assign_exc_cb()
6576 		if (strcmp(prog->name, btf__str_by_offset(obj->btf, t->name_off)) != 0)  in bpf_prog_assign_exc_cb()
6582 		if (prog->exception_cb_idx >= 0) {  in bpf_prog_assign_exc_cb()
6583 			prog->exception_cb_idx = -1;  in bpf_prog_assign_exc_cb()
6590 				prog->name);  in bpf_prog_assign_exc_cb()
6591 			return -EINVAL;  in bpf_prog_assign_exc_cb()
6594 		for (j = 0; j < obj->nr_programs; j++) {  in bpf_prog_assign_exc_cb()
6595 			struct bpf_program *subprog = &obj->programs[j];  in bpf_prog_assign_exc_cb()
6599 			if (strcmp(name, subprog->name) != 0)  in bpf_prog_assign_exc_cb()
6601 			/* Enforce non-hidden, as from verifier point of  in bpf_prog_assign_exc_cb()
6605 			if (!subprog->sym_global || subprog->mark_btf_static) {  in bpf_prog_assign_exc_cb()
6606 				pr_warn("prog '%s': exception callback %s must be a global non-hidden function\n",  in bpf_prog_assign_exc_cb()
6607 					prog->name, subprog->name);  in bpf_prog_assign_exc_cb()
6608 				return -EINVAL;  in bpf_prog_assign_exc_cb()
6611 			if (prog->exception_cb_idx >= 0) {  in bpf_prog_assign_exc_cb()
6613 					prog->name, subprog->name);  in bpf_prog_assign_exc_cb()
6614 				return -EINVAL;  in bpf_prog_assign_exc_cb()
6616 			prog->exception_cb_idx = j;  in bpf_prog_assign_exc_cb()
6620 		if (prog->exception_cb_idx >= 0)  in bpf_prog_assign_exc_cb()
6623 		pr_warn("prog '%s': cannot find exception callback '%s'\n", prog->name, name);  in bpf_prog_assign_exc_cb()
6624 		return -ENOENT;  in bpf_prog_assign_exc_cb()
6662 /* forward declarations for arch-specific underlying types of bpf_user_pt_regs_t typedef,
6664  * with this approach we don't need any extra arch-specific #ifdef guards
6687 		t = btf__type_by_id(btf, t->type);  in need_func_arg_type_fixup()
6689 	    (prog->type == BPF_PROG_TYPE_KPROBE || prog->type == BPF_PROG_TYPE_PERF_EVENT)) {  in need_func_arg_type_fixup()
6690 		tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";  in need_func_arg_type_fixup()
6696 	t = skip_mods_and_typedefs(btf, t->type, NULL);  in need_func_arg_type_fixup()
6703 	tname = btf__str_by_offset(btf, t->name_off) ?: "<anon>";  in need_func_arg_type_fixup()
6708 	switch (prog->type) {  in need_func_arg_type_fixup()
6728 		if (btf_is_int(t) && t->size == 8)  in need_func_arg_type_fixup()
6737 		prog->name, subprog_name, arg_idx, ctx_name);  in need_func_arg_type_fixup()
6748 	/* caller already validated FUNC -> FUNC_PROTO validity */  in clone_func_btf_info()
6750 	fn_proto_t = btf_type_by_id(btf, fn_t->type);  in clone_func_btf_info()
6761 	fn_name_off = fn_t->name_off; /* we are about to invalidate fn_t */  in clone_func_btf_info()
6763 	orig_proto_id = fn_t->type; /* original FUNC_PROTO ID */  in clone_func_btf_info()
6764 	ret_type_id = fn_proto_t->type; /* fn_proto_t will be invalidated */  in clone_func_btf_info()
6770 		return -EINVAL;  in clone_func_btf_info()
6778 		name_off = p->name_off;  in clone_func_btf_info()
6780 		err = btf__add_func_param(btf, "", p->type);  in clone_func_btf_info()
6786 		p->name_off = name_off; /* use remembered str offset */  in clone_func_btf_info()
6789 	/* clone FUNC now, btf__add_func() enforces non-empty name, so use  in clone_func_btf_info()
6793 	fn_id = btf__add_func(btf, prog->name, linkage, fn_proto_id);  in clone_func_btf_info()
6795 		return -EINVAL;  in clone_func_btf_info()
6798 	fn_t->name_off = fn_name_off; /* reuse original string */  in clone_func_btf_info()
6815 	struct btf *btf = obj->btf;  in bpf_program_fixup_func_info()
6822 	/* no .BTF.ext, no problem */  in bpf_program_fixup_func_info()
6823 	if (!obj->btf_ext || !prog->func_info)  in bpf_program_fixup_func_info()
6834 		if (global_ctx_map[i].prog_type != prog->type)  in bpf_program_fixup_func_info()
6843 	orig_ids = calloc(prog->func_info_cnt, sizeof(*orig_ids));  in bpf_program_fixup_func_info()
6845 		return -ENOMEM;  in bpf_program_fixup_func_info()
6846 	for (i = 0; i < prog->func_info_cnt; i++) {  in bpf_program_fixup_func_info()
6847 		func_rec = prog->func_info + prog->func_info_rec_size * i;  in bpf_program_fixup_func_info()
6848 		orig_ids[i] = func_rec->type_id;  in bpf_program_fixup_func_info()
6853 	 * clone and adjust FUNC -> FUNC_PROTO combo  in bpf_program_fixup_func_info()
6860 		if (strcmp(btf__str_by_offset(btf, t->name_off), ctx_tag) != 0)  in bpf_program_fixup_func_info()
6864 		orig_fn_id = t->type;  in bpf_program_fixup_func_info()
6869 		/* sanity check FUNC -> FUNC_PROTO chain, just in case */  in bpf_program_fixup_func_info()
6870 		fn_proto_t = btf_type_by_id(btf, fn_t->type);  in bpf_program_fixup_func_info()
6876 		for (rec_idx = 0; rec_idx < prog->func_info_cnt; rec_idx++) {  in bpf_program_fixup_func_info()
6877 			if (orig_ids[rec_idx] == t->type) {  in bpf_program_fixup_func_info()
6878 				func_rec = prog->func_info + prog->func_info_rec_size * rec_idx;  in bpf_program_fixup_func_info()
6888 		arg_idx = btf_decl_tag(t)->component_idx;  in bpf_program_fixup_func_info()
6894 		fn_name = btf__str_by_offset(btf, fn_t->name_off) ?: "<anon>";  in bpf_program_fixup_func_info()
6895 		if (!need_func_arg_type_fixup(btf, prog, fn_name, arg_idx, p->type, ctx_name))  in bpf_program_fixup_func_info()
6899 		if (func_rec->type_id == orig_fn_id) {  in bpf_program_fixup_func_info()
6909 			func_rec->type_id = fn_id;  in bpf_program_fixup_func_info()
6912 		/* create PTR -> STRUCT type chain to mark PTR_TO_CTX argument;  in bpf_program_fixup_func_info()
6914 		 * funcs share the same program type, so need only PTR ->  in bpf_program_fixup_func_info()
6921 				err = -EINVAL;  in bpf_program_fixup_func_info()
6927 		tag_id = btf__add_decl_tag(btf, ctx_tag, func_rec->type_id, arg_idx);  in bpf_program_fixup_func_info()
6929 			err = -EINVAL;  in bpf_program_fixup_func_info()
6934 		fn_t = btf_type_by_id(btf, func_rec->type_id);  in bpf_program_fixup_func_info()
6935 		fn_proto_t = btf_type_by_id(btf, fn_t->type);  in bpf_program_fixup_func_info()
6939 		p->type = ptr_id;  in bpf_program_fixup_func_info()
6955 	if (obj->btf_ext) {  in bpf_object__relocate()
6958 			pr_warn("failed to perform CO-RE relocations: %d\n",  in bpf_object__relocate()
6965 	/* Before relocating calls pre-process relocations and mark  in bpf_object__relocate()
6972 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__relocate()
6973 		prog = &obj->programs[i];  in bpf_object__relocate()
6974 		for (j = 0; j < prog->nr_reloc; j++) {  in bpf_object__relocate()
6975 			struct reloc_desc *relo = &prog->reloc_desc[j];  in bpf_object__relocate()
6976 			struct bpf_insn *insn = &prog->insns[relo->insn_idx];  in bpf_object__relocate()
6979 			if (relo->type == RELO_SUBPROG_ADDR)  in bpf_object__relocate()
6991 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__relocate()
6992 		prog = &obj->programs[i];  in bpf_object__relocate()
6993 		/* sub-program's sub-calls are relocated within the context of  in bpf_object__relocate()
6998 		if (!prog->autoload)  in bpf_object__relocate()
7004 				prog->name, err);  in bpf_object__relocate()
7012 		if (prog->exception_cb_idx >= 0) {  in bpf_object__relocate()
7013 			struct bpf_program *subprog = &obj->programs[prog->exception_cb_idx];  in bpf_object__relocate()
7020 			if (subprog->sub_insn_off == 0) {  in bpf_object__relocate()
7030 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__relocate()
7031 		prog = &obj->programs[i];  in bpf_object__relocate()
7034 		if (!prog->autoload)  in bpf_object__relocate()
7041 				prog->name, err);  in bpf_object__relocate()
7045 		/* Fix up .BTF.ext information, if necessary */  in bpf_object__relocate()
7048 			pr_warn("prog '%s': failed to perform .BTF.ext fix ups: %d\n",  in bpf_object__relocate()
7049 				prog->name, err);  in bpf_object__relocate()
7077 	if (!obj->efile.btf_maps_sec_btf_id || !obj->btf)  in bpf_object__collect_map_relos()
7078 		return -EINVAL;  in bpf_object__collect_map_relos()
7079 	sec = btf__type_by_id(obj->btf, obj->efile.btf_maps_sec_btf_id);  in bpf_object__collect_map_relos()
7081 		return -EINVAL;  in bpf_object__collect_map_relos()
7083 	nrels = shdr->sh_size / shdr->sh_entsize;  in bpf_object__collect_map_relos()
7088 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_map_relos()
7091 		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));  in bpf_object__collect_map_relos()
7094 				i, (size_t)ELF64_R_SYM(rel->r_info));  in bpf_object__collect_map_relos()
7095 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_map_relos()
7097 		name = elf_sym_str(obj, sym->st_name) ?: "<?>";  in bpf_object__collect_map_relos()
7099 		pr_debug(".maps relo #%d: for %zd value %zd rel->r_offset %zu name %d ('%s')\n",  in bpf_object__collect_map_relos()
7100 			 i, (ssize_t)(rel->r_info >> 32), (size_t)sym->st_value,  in bpf_object__collect_map_relos()
7101 			 (size_t)rel->r_offset, sym->st_name, name);  in bpf_object__collect_map_relos()
7103 		for (j = 0; j < obj->nr_maps; j++) {  in bpf_object__collect_map_relos()
7104 			map = &obj->maps[j];  in bpf_object__collect_map_relos()
7105 			if (map->sec_idx != obj->efile.btf_maps_shndx)  in bpf_object__collect_map_relos()
7108 			vi = btf_var_secinfos(sec) + map->btf_var_idx;  in bpf_object__collect_map_relos()
7109 			if (vi->offset <= rel->r_offset &&  in bpf_object__collect_map_relos()
7110 			    rel->r_offset + bpf_ptr_sz <= vi->offset + vi->size)  in bpf_object__collect_map_relos()
7113 		if (j == obj->nr_maps) {  in bpf_object__collect_map_relos()
7114 			pr_warn(".maps relo #%d: cannot find map '%s' at rel->r_offset %zu\n",  in bpf_object__collect_map_relos()
7115 				i, name, (size_t)rel->r_offset);  in bpf_object__collect_map_relos()
7116 			return -EINVAL;  in bpf_object__collect_map_relos()
7119 		is_map_in_map = bpf_map_type__is_map_in_map(map->def.type);  in bpf_object__collect_map_relos()
7120 		is_prog_array = map->def.type == BPF_MAP_TYPE_PROG_ARRAY;  in bpf_object__collect_map_relos()
7123 			if (sym->st_shndx != obj->efile.btf_maps_shndx) {  in bpf_object__collect_map_relos()
7124 				pr_warn(".maps relo #%d: '%s' isn't a BTF-defined map\n",  in bpf_object__collect_map_relos()
7126 				return -LIBBPF_ERRNO__RELOC;  in bpf_object__collect_map_relos()
7128 			if (map->def.type == BPF_MAP_TYPE_HASH_OF_MAPS &&  in bpf_object__collect_map_relos()
7129 			    map->def.key_size != sizeof(int)) {  in bpf_object__collect_map_relos()
7130 				pr_warn(".maps relo #%d: hash-of-maps '%s' should have key size %zu.\n",  in bpf_object__collect_map_relos()
7131 					i, map->name, sizeof(int));  in bpf_object__collect_map_relos()
7132 				return -EINVAL;  in bpf_object__collect_map_relos()
7138 				return -ESRCH;  in bpf_object__collect_map_relos()
7145 				return -ESRCH;  in bpf_object__collect_map_relos()
7147 			if (targ_prog->sec_idx != sym->st_shndx ||  in bpf_object__collect_map_relos()
7148 			    targ_prog->sec_insn_off * 8 != sym->st_value ||  in bpf_object__collect_map_relos()
7150 				pr_warn(".maps relo #%d: '%s' isn't an entry-point program\n",  in bpf_object__collect_map_relos()
7152 				return -LIBBPF_ERRNO__RELOC;  in bpf_object__collect_map_relos()
7155 			return -EINVAL;  in bpf_object__collect_map_relos()
7158 		var = btf__type_by_id(obj->btf, vi->type);  in bpf_object__collect_map_relos()
7159 		def = skip_mods_and_typedefs(obj->btf, var->type, NULL);  in bpf_object__collect_map_relos()
7161 			return -EINVAL;  in bpf_object__collect_map_relos()
7162 		member = btf_members(def) + btf_vlen(def) - 1;  in bpf_object__collect_map_relos()
7163 		mname = btf__name_by_offset(obj->btf, member->name_off);  in bpf_object__collect_map_relos()
7165 			return -EINVAL;  in bpf_object__collect_map_relos()
7167 		moff = btf_member_bit_offset(def, btf_vlen(def) - 1) / 8;  in bpf_object__collect_map_relos()
7168 		if (rel->r_offset - vi->offset < moff)  in bpf_object__collect_map_relos()
7169 			return -EINVAL;  in bpf_object__collect_map_relos()
7171 		moff = rel->r_offset - vi->offset - moff;  in bpf_object__collect_map_relos()
7176 			return -EINVAL;  in bpf_object__collect_map_relos()
7178 		if (moff >= map->init_slots_sz) {  in bpf_object__collect_map_relos()
7180 			tmp = libbpf_reallocarray(map->init_slots, new_sz, host_ptr_sz);  in bpf_object__collect_map_relos()
7182 				return -ENOMEM;  in bpf_object__collect_map_relos()
7183 			map->init_slots = tmp;  in bpf_object__collect_map_relos()
7184 			memset(map->init_slots + map->init_slots_sz, 0,  in bpf_object__collect_map_relos()
7185 			       (new_sz - map->init_slots_sz) * host_ptr_sz);  in bpf_object__collect_map_relos()
7186 			map->init_slots_sz = new_sz;  in bpf_object__collect_map_relos()
7188 		map->init_slots[moff] = is_map_in_map ? (void *)targ_map : (void *)targ_prog;  in bpf_object__collect_map_relos()
7191 			 i, map->name, moff, type, name);  in bpf_object__collect_map_relos()
7201 	for (i = 0; i < obj->efile.sec_cnt; i++) {  in bpf_object__collect_relos()
7202 		struct elf_sec_desc *sec_desc = &obj->efile.secs[i];  in bpf_object__collect_relos()
7207 		if (sec_desc->sec_type != SEC_RELO)  in bpf_object__collect_relos()
7210 		shdr = sec_desc->shdr;  in bpf_object__collect_relos()
7211 		data = sec_desc->data;  in bpf_object__collect_relos()
7212 		idx = shdr->sh_info;  in bpf_object__collect_relos()
7214 		if (shdr->sh_type != SHT_REL || idx < 0 || idx >= obj->efile.sec_cnt) {  in bpf_object__collect_relos()
7216 			return -LIBBPF_ERRNO__INTERNAL;  in bpf_object__collect_relos()
7219 		if (obj->efile.secs[idx].sec_type == SEC_ST_OPS)  in bpf_object__collect_relos()
7221 		else if (idx == obj->efile.btf_maps_shndx)  in bpf_object__collect_relos()
7235 	if (BPF_CLASS(insn->code) == BPF_JMP &&  in insn_is_helper_call()
7236 	    BPF_OP(insn->code) == BPF_CALL &&  in insn_is_helper_call()
7237 	    BPF_SRC(insn->code) == BPF_K &&  in insn_is_helper_call()
7238 	    insn->src_reg == 0 &&  in insn_is_helper_call()
7239 	    insn->dst_reg == 0) {  in insn_is_helper_call()
7240 		    *func_id = insn->imm;  in insn_is_helper_call()
7248 	struct bpf_insn *insn = prog->insns;  in bpf_object__sanitize_prog()
7252 	if (obj->gen_loader)  in bpf_object__sanitize_prog()
7255 	for (i = 0; i < prog->insns_cnt; i++, insn++) {  in bpf_object__sanitize_prog()
7267 				insn->imm = BPF_FUNC_probe_read;  in bpf_object__sanitize_prog()
7272 				insn->imm = BPF_FUNC_probe_read_str;  in bpf_object__sanitize_prog()
7284 /* this is called as prog->sec_def->prog_prepare_load_fn for libbpf-supported sec_defs */
7291 	if ((def & SEC_EXP_ATTACH_OPT) && !kernel_supports(prog->obj, FEAT_EXP_ATTACH_TYPE))  in libbpf_prepare_prog_load()
7292 		opts->expected_attach_type = 0;  in libbpf_prepare_prog_load()
7295 		opts->prog_flags |= BPF_F_SLEEPABLE;  in libbpf_prepare_prog_load()
7297 	if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS))  in libbpf_prepare_prog_load()
7298 		opts->prog_flags |= BPF_F_XDP_HAS_FRAGS;  in libbpf_prepare_prog_load()
7301 	if ((def & SEC_USDT) && kernel_supports(prog->obj, FEAT_UPROBE_MULTI_LINK))  in libbpf_prepare_prog_load()
7302 		prog->expected_attach_type = BPF_TRACE_UPROBE_MULTI;  in libbpf_prepare_prog_load()
7304 	if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) {  in libbpf_prepare_prog_load()
7308 		attach_name = strchr(prog->sec_name, '/');  in libbpf_prepare_prog_load()
7319 …pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\…  in libbpf_prepare_prog_load()
7320 				prog->name);  in libbpf_prepare_prog_load()
7321 			return -EINVAL;  in libbpf_prepare_prog_load()
7330 		prog->attach_btf_obj_fd = btf_obj_fd;  in libbpf_prepare_prog_load()
7331 		prog->attach_btf_id = btf_type_id;  in libbpf_prepare_prog_load()
7334 		 * prog->atach_btf_obj_fd/prog->attach_btf_id anymore because  in libbpf_prepare_prog_load()
7338 		opts->attach_btf_obj_fd = btf_obj_fd;  in libbpf_prepare_prog_load()
7339 		opts->attach_btf_id = btf_type_id;  in libbpf_prepare_prog_load()
7356 	__u32 log_level = prog->log_level;  in bpf_object_load_prog()
7362 	switch (prog->type) {  in bpf_object_load_prog()
7369 			prog->name, prog->sec_name);  in bpf_object_load_prog()
7370 		return -EINVAL;  in bpf_object_load_prog()
7372 		if (prog->attach_btf_id == 0) {  in bpf_object_load_prog()
7374 				prog->name);  in bpf_object_load_prog()
7375 			return -EINVAL;  in bpf_object_load_prog()
7383 		return -EINVAL;  in bpf_object_load_prog()
7386 		prog_name = prog->name;  in bpf_object_load_prog()
7387 	load_attr.attach_prog_fd = prog->attach_prog_fd;  in bpf_object_load_prog()
7388 	load_attr.attach_btf_obj_fd = prog->attach_btf_obj_fd;  in bpf_object_load_prog()
7389 	load_attr.attach_btf_id = prog->attach_btf_id;  in bpf_object_load_prog()
7391 	load_attr.prog_ifindex = prog->prog_ifindex;  in bpf_object_load_prog()
7394 	if (obj->btf && btf__fd(obj->btf) >= 0 && kernel_supports(obj, FEAT_BTF_FUNC)) {  in bpf_object_load_prog()
7395 		load_attr.prog_btf_fd = btf__fd(obj->btf);  in bpf_object_load_prog()
7396 		load_attr.func_info = prog->func_info;  in bpf_object_load_prog()
7397 		load_attr.func_info_rec_size = prog->func_info_rec_size;  in bpf_object_load_prog()
7398 		load_attr.func_info_cnt = prog->func_info_cnt;  in bpf_object_load_prog()
7399 		load_attr.line_info = prog->line_info;  in bpf_object_load_prog()
7400 		load_attr.line_info_rec_size = prog->line_info_rec_size;  in bpf_object_load_prog()
7401 		load_attr.line_info_cnt = prog->line_info_cnt;  in bpf_object_load_prog()
7404 	load_attr.prog_flags = prog->prog_flags;  in bpf_object_load_prog()
7405 	load_attr.fd_array = obj->fd_array;  in bpf_object_load_prog()
7407 	load_attr.token_fd = obj->token_fd;  in bpf_object_load_prog()
7408 	if (obj->token_fd)  in bpf_object_load_prog()
7412 	if (prog->sec_def && prog->sec_def->prog_prepare_load_fn) {  in bpf_object_load_prog()
7413 		err = prog->sec_def->prog_prepare_load_fn(prog, &load_attr, prog->sec_def->cookie);  in bpf_object_load_prog()
7416 				prog->name, err);  in bpf_object_load_prog()
7419 		insns = prog->insns;  in bpf_object_load_prog()
7420 		insns_cnt = prog->insns_cnt;  in bpf_object_load_prog()
7424 	load_attr.expected_attach_type = prog->expected_attach_type;  in bpf_object_load_prog()
7426 	if (obj->gen_loader) {  in bpf_object_load_prog()
7427 		bpf_gen__prog_load(obj->gen_loader, prog->type, prog->name,  in bpf_object_load_prog()
7429 				   prog - obj->programs);  in bpf_object_load_prog()
7430 		*prog_fd = -1;  in bpf_object_load_prog()
7441 		if (prog->log_buf) {  in bpf_object_load_prog()
7442 			log_buf = prog->log_buf;  in bpf_object_load_prog()
7443 			log_buf_size = prog->log_size;  in bpf_object_load_prog()
7445 		} else if (obj->log_buf) {  in bpf_object_load_prog()
7446 			log_buf = obj->log_buf;  in bpf_object_load_prog()
7447 			log_buf_size = obj->log_size;  in bpf_object_load_prog()
7453 				ret = -ENOMEM;  in bpf_object_load_prog()
7466 	ret = bpf_prog_load(prog->type, prog_name, license, insns, insns_cnt, &load_attr);  in bpf_object_load_prog()
7469 			pr_debug("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",  in bpf_object_load_prog()
7470 				 prog->name, log_buf);  in bpf_object_load_prog()
7473 		if (obj->has_rodata && kernel_supports(obj, FEAT_PROG_BIND_MAP)) {  in bpf_object_load_prog()
7477 			for (i = 0; i < obj->nr_maps; i++) {  in bpf_object_load_prog()
7478 				map = &prog->obj->maps[i];  in bpf_object_load_prog()
7479 				if (map->libbpf_type != LIBBPF_MAP_RODATA)  in bpf_object_load_prog()
7482 				if (bpf_prog_bind_map(ret, map->fd, NULL)) {  in bpf_object_load_prog()
7485 						prog->name, map->real_name, cp);  in bpf_object_load_prog()
7505 	 * Currently, we'll get -EINVAL when we reach (UINT_MAX >> 2).  in bpf_object_load_prog()
7510 	ret = -errno;  in bpf_object_load_prog()
7512 	/* post-process verifier log to improve error descriptions */  in bpf_object_load_prog()
7516 	pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp);  in bpf_object_load_prog()
7520 		pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n",  in bpf_object_load_prog()
7521 			prog->name, log_buf);  in bpf_object_load_prog()
7537 	p = cur - 1;  in find_prev_line()
7538 	while (p - 1 >= buf && *(p - 1) != '\n')  in find_prev_line()
7539 		p--;  in find_prev_line()
7547 	/* size of the remaining log content to the right from the to-be-replaced part */  in patch_log()
7548 	size_t rem_sz = (buf + log_sz) - (orig + orig_sz);  in patch_log()
7553 		 * shift log contents by (patch_sz - orig_sz) bytes to the right  in patch_log()
7554 		 * starting from after to-be-replaced part of the log.  in patch_log()
7557 		 * shift log contents by (orig_sz - patch_sz) bytes to the left  in patch_log()
7558 		 * starting from after to-be-replaced part of the log  in patch_log()
7567 				patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1;  in patch_log()
7569 			} else if (patch_sz - orig_sz > buf_sz - log_sz) {  in patch_log()
7571 				rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz);  in patch_log()
7585 	/* Expected log for failed and not properly guarded CO-RE relocation:  in fixup_log_failed_core_relo()
7586 	 * line1 -> 123: (85) call unknown#195896080  in fixup_log_failed_core_relo()
7587 	 * line2 -> invalid func unknown#195896080  in fixup_log_failed_core_relo()
7588 	 * line3 -> <anything else or end of buffer>  in fixup_log_failed_core_relo()
7591 	 * instruction index to find corresponding CO-RE relocation and  in fixup_log_failed_core_relo()
7593 	 * failed CO-RE relocation.  in fixup_log_failed_core_relo()
7607 	err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec);  in fixup_log_failed_core_relo()
7613 		 "%d: <invalid CO-RE relocation>\n"  in fixup_log_failed_core_relo()
7614 		 "failed to resolve CO-RE relocation %s%s\n",  in fixup_log_failed_core_relo()
7617 	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);  in fixup_log_failed_core_relo()
7625 	 * line1 -> 123: (85) call unknown#2001000345  in fixup_log_missing_map_load()
7626 	 * line2 -> invalid func unknown#2001000345  in fixup_log_missing_map_load()
7627 	 * line3 -> <anything else or end of buffer>  in fixup_log_missing_map_load()
7630 	 * "345" in "2001000345" is a map index in obj->maps to fetch map name.  in fixup_log_missing_map_load()
7632 	struct bpf_object *obj = prog->obj;  in fixup_log_missing_map_load()
7640 	map_idx -= POISON_LDIMM64_MAP_BASE;  in fixup_log_missing_map_load()
7641 	if (map_idx < 0 || map_idx >= obj->nr_maps)  in fixup_log_missing_map_load()
7643 	map = &obj->maps[map_idx];  in fixup_log_missing_map_load()
7648 		 insn_idx, map->name);  in fixup_log_missing_map_load()
7650 	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);  in fixup_log_missing_map_load()
7658 	 * line1 -> 123: (85) call unknown#2002000345  in fixup_log_missing_kfunc_call()
7659 	 * line2 -> invalid func unknown#2002000345  in fixup_log_missing_kfunc_call()
7660 	 * line3 -> <anything else or end of buffer>  in fixup_log_missing_kfunc_call()
7663 	 * "345" in "2002000345" is an extern index in obj->externs to fetch kfunc name.  in fixup_log_missing_kfunc_call()
7665 	struct bpf_object *obj = prog->obj;  in fixup_log_missing_kfunc_call()
7666 	const struct extern_desc *ext;  in fixup_log_missing_kfunc_call()  local
7673 	ext_idx -= POISON_CALL_KFUNC_BASE;  in fixup_log_missing_kfunc_call()
7674 	if (ext_idx < 0 || ext_idx >= obj->nr_extern)  in fixup_log_missing_kfunc_call()
7676 	ext = &obj->externs[ext_idx];  in fixup_log_missing_kfunc_call()
7681 		 insn_idx, ext->name);  in fixup_log_missing_kfunc_call()
7683 	patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch);  in fixup_log_missing_kfunc_call()
7698 	next_line = buf + log_sz - 1;  in fixup_verifier_log()
7710 			/* failed CO-RE relocation case */  in fixup_verifier_log()
7738 	struct bpf_object *obj = prog->obj;  in bpf_program_record_relos()
7741 	for (i = 0; i < prog->nr_reloc; i++) {  in bpf_program_record_relos()
7742 		struct reloc_desc *relo = &prog->reloc_desc[i];  in bpf_program_record_relos()
7743 		struct extern_desc *ext = &obj->externs[relo->ext_idx];  in bpf_program_record_relos()  local
7746 		switch (relo->type) {  in bpf_program_record_relos()
7748 			if (ext->type != EXT_KSYM)  in bpf_program_record_relos()
7750 			kind = btf_is_var(btf__type_by_id(obj->btf, ext->btf_id)) ?  in bpf_program_record_relos()
7752 			bpf_gen__record_extern(obj->gen_loader, ext->name,  in bpf_program_record_relos()
7753 					       ext->is_weak, !ext->ksym.type_id,  in bpf_program_record_relos()
7754 					       true, kind, relo->insn_idx);  in bpf_program_record_relos()
7757 			bpf_gen__record_extern(obj->gen_loader, ext->name,  in bpf_program_record_relos()
7758 					       ext->is_weak, false, false, BTF_KIND_FUNC,  in bpf_program_record_relos()
7759 					       relo->insn_idx);  in bpf_program_record_relos()
7763 				.insn_off = relo->insn_idx * 8,  in bpf_program_record_relos()
7764 				.type_id = relo->core_relo->type_id,  in bpf_program_record_relos()
7765 				.access_str_off = relo->core_relo->access_str_off,  in bpf_program_record_relos()
7766 				.kind = relo->core_relo->kind,  in bpf_program_record_relos()
7769 			bpf_gen__record_relo_core(obj->gen_loader, &cr);  in bpf_program_record_relos()
7786 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__load_progs()
7787 		prog = &obj->programs[i];  in bpf_object__load_progs()
7793 	for (i = 0; i < obj->nr_programs; i++) {  in bpf_object__load_progs()
7794 		prog = &obj->programs[i];  in bpf_object__load_progs()
7797 		if (!prog->autoload) {  in bpf_object__load_progs()
7798 			pr_debug("prog '%s': skipped loading\n", prog->name);  in bpf_object__load_progs()
7801 		prog->log_level |= log_level;  in bpf_object__load_progs()
7803 		if (obj->gen_loader)  in bpf_object__load_progs()
7806 		err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,  in bpf_object__load_progs()
7807 					   obj->license, obj->kern_version, &prog->fd);  in bpf_object__load_progs()
7809 			pr_warn("prog '%s': failed to load: %d\n", prog->name, err);  in bpf_object__load_progs()
7826 		prog->sec_def = find_sec_def(prog->sec_name);  in bpf_object_init_progs()
7827 		if (!prog->sec_def) {  in bpf_object_init_progs()
7830 				prog->name, prog->sec_name);  in bpf_object_init_progs()
7834 		prog->type = prog->sec_def->prog_type;  in bpf_object_init_progs()
7835 		prog->expected_attach_type = prog->sec_def->expected_attach_type;  in bpf_object_init_progs()
7840 		if (prog->sec_def->prog_setup_fn) {  in bpf_object_init_progs()
7841 			err = prog->sec_def->prog_setup_fn(prog, prog->sec_def->cookie);  in bpf_object_init_progs()
7844 					prog->name, err);  in bpf_object_init_progs()
7856 	const char *obj_name, *kconfig, *btf_tmp_path, *token_path;  in bpf_object_open()  local
7867 		return ERR_PTR(-LIBBPF_ERRNO__LIBELF);  in bpf_object_open()
7871 		return ERR_PTR(-EINVAL);  in bpf_object_open()
7876 			snprintf(tmp_name, sizeof(tmp_name), "%lx-%lx",  in bpf_object_open()
7889 		return ERR_PTR(-EINVAL);  in bpf_object_open()
7891 		return ERR_PTR(-EINVAL);  in bpf_object_open()
7901 		return ERR_PTR(-ENAMETOOLONG);  in bpf_object_open()
7907 	obj->log_buf = log_buf;  in bpf_object_open()
7908 	obj->log_size = log_size;  in bpf_object_open()
7909 	obj->log_level = log_level;  in bpf_object_open()
7912 		obj->token_path = strdup(token_path);  in bpf_object_open()
7913 		if (!obj->token_path) {  in bpf_object_open()
7914 			err = -ENOMEM;  in bpf_object_open()
7922 			err = -ENAMETOOLONG;  in bpf_object_open()
7925 		obj->btf_custom_path = strdup(btf_tmp_path);  in bpf_object_open()
7926 		if (!obj->btf_custom_path) {  in bpf_object_open()
7927 			err = -ENOMEM;  in bpf_object_open()
7932 	kconfig = OPTS_GET(opts, kconfig, NULL);  in bpf_object_open()
7933 	if (kconfig) {  in bpf_object_open()
7934 		obj->kconfig = strdup(kconfig);  in bpf_object_open()
7935 		if (!obj->kconfig) {  in bpf_object_open()
7936 			err = -ENOMEM;  in bpf_object_open()
7964 		return libbpf_err_ptr(-EINVAL);  in bpf_object__open_file()
7981 		return libbpf_err_ptr(-EINVAL);  in bpf_object__open_mem()
7991 		return libbpf_err(-EINVAL);  in bpf_object_unload()
7993 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object_unload()
7994 		zclose(obj->maps[i].fd);  in bpf_object_unload()
7995 		if (obj->maps[i].st_ops)  in bpf_object_unload()
7996 			zfree(&obj->maps[i].st_ops->kern_vdata);  in bpf_object_unload()
7999 	for (i = 0; i < obj->nr_programs; i++)  in bpf_object_unload()
8000 		bpf_program__unload(&obj->programs[i]);  in bpf_object_unload()
8013 			m->def.map_flags &= ~BPF_F_MMAPABLE;  in bpf_object__sanitize_maps()
8028 		err = -errno;  in libbpf_kallsyms_parse()
8040 			err = -EINVAL;  in libbpf_kallsyms_parse()
8058 	struct extern_desc *ext;  in kallsyms_cb()  local
8060 	ext = find_extern_by_name(obj, sym_name);  in kallsyms_cb()
8061 	if (!ext || ext->type != EXT_KSYM)  in kallsyms_cb()
8064 	t = btf__type_by_id(obj->btf, ext->btf_id);  in kallsyms_cb()
8068 	if (ext->is_set && ext->ksym.addr != sym_addr) {  in kallsyms_cb()
8070 			sym_name, ext->ksym.addr, sym_addr);  in kallsyms_cb()
8071 		return -EINVAL;  in kallsyms_cb()
8073 	if (!ext->is_set) {  in kallsyms_cb()
8074 		ext->is_set = true;  in kallsyms_cb()
8075 		ext->ksym.addr = sym_addr;  in kallsyms_cb()
8094 	btf = obj->btf_vmlinux;  in find_ksym_btf_id()
8098 	if (id == -ENOENT) {  in find_ksym_btf_id()
8103 		for (i = 0; i < obj->btf_module_cnt; i++) {  in find_ksym_btf_id()
8105 			mod_btf = &obj->btf_modules[i];  in find_ksym_btf_id()
8106 			btf = mod_btf->btf;  in find_ksym_btf_id()
8108 			if (id != -ENOENT)  in find_ksym_btf_id()
8113 		return -ESRCH;  in find_ksym_btf_id()
8121 					       struct extern_desc *ext)  in bpf_object__resolve_ksym_var_btf_id()  argument
8130 	id = find_ksym_btf_id(obj, ext->name, BTF_KIND_VAR, &btf, &mod_btf);  in bpf_object__resolve_ksym_var_btf_id()
8132 		if (id == -ESRCH && ext->is_weak)  in bpf_object__resolve_ksym_var_btf_id()
8135 			ext->name);  in bpf_object__resolve_ksym_var_btf_id()
8140 	local_type_id = ext->ksym.type_id;  in bpf_object__resolve_ksym_var_btf_id()
8144 	targ_var_name = btf__name_by_offset(btf, targ_var->name_off);  in bpf_object__resolve_ksym_var_btf_id()
8145 	targ_type = skip_mods_and_typedefs(btf, targ_var->type, &targ_type_id);  in bpf_object__resolve_ksym_var_btf_id()
8147 	err = bpf_core_types_are_compat(obj->btf, local_type_id,  in bpf_object__resolve_ksym_var_btf_id()
8153 		local_type = btf__type_by_id(obj->btf, local_type_id);  in bpf_object__resolve_ksym_var_btf_id()
8154 		local_name = btf__name_by_offset(obj->btf, local_type->name_off);  in bpf_object__resolve_ksym_var_btf_id()
8155 		targ_name = btf__name_by_offset(btf, targ_type->name_off);  in bpf_object__resolve_ksym_var_btf_id()
8158 			ext->name, local_type_id,  in bpf_object__resolve_ksym_var_btf_id()
8161 		return -EINVAL;  in bpf_object__resolve_ksym_var_btf_id()
8164 	ext->is_set = true;  in bpf_object__resolve_ksym_var_btf_id()
8165 	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;  in bpf_object__resolve_ksym_var_btf_id()
8166 	ext->ksym.kernel_btf_id = id;  in bpf_object__resolve_ksym_var_btf_id()
8168 		 ext->name, id, btf_kind_str(targ_var), targ_var_name);  in bpf_object__resolve_ksym_var_btf_id()
8174 						struct extern_desc *ext)  in bpf_object__resolve_ksym_func_btf_id()  argument
8182 	local_func_proto_id = ext->ksym.type_id;  in bpf_object__resolve_ksym_func_btf_id()
8184 	kfunc_id = find_ksym_btf_id(obj, ext->essent_name ?: ext->name, BTF_KIND_FUNC, &kern_btf,  in bpf_object__resolve_ksym_func_btf_id()
8187 		if (kfunc_id == -ESRCH && ext->is_weak)  in bpf_object__resolve_ksym_func_btf_id()
8190 			ext->name);  in bpf_object__resolve_ksym_func_btf_id()
8195 	kfunc_proto_id = kern_func->type;  in bpf_object__resolve_ksym_func_btf_id()
8197 	ret = bpf_core_types_are_compat(obj->btf, local_func_proto_id,  in bpf_object__resolve_ksym_func_btf_id()
8200 		if (ext->is_weak)  in bpf_object__resolve_ksym_func_btf_id()
8204 			ext->name, local_func_proto_id,  in bpf_object__resolve_ksym_func_btf_id()
8205 			mod_btf ? mod_btf->name : "vmlinux", kfunc_proto_id);  in bpf_object__resolve_ksym_func_btf_id()
8206 		return -EINVAL;  in bpf_object__resolve_ksym_func_btf_id()
8210 	if (mod_btf && !mod_btf->fd_array_idx) {  in bpf_object__resolve_ksym_func_btf_id()
8211 		/* insn->off is s16 */  in bpf_object__resolve_ksym_func_btf_id()
8212 		if (obj->fd_array_cnt == INT16_MAX) {  in bpf_object__resolve_ksym_func_btf_id()
8214 				ext->name, mod_btf->fd_array_idx);  in bpf_object__resolve_ksym_func_btf_id()
8215 			return -E2BIG;  in bpf_object__resolve_ksym_func_btf_id()
8218 		if (!obj->fd_array_cnt)  in bpf_object__resolve_ksym_func_btf_id()
8219 			obj->fd_array_cnt = 1;  in bpf_object__resolve_ksym_func_btf_id()
8221 		ret = libbpf_ensure_mem((void **)&obj->fd_array, &obj->fd_array_cap, sizeof(int),  in bpf_object__resolve_ksym_func_btf_id()
8222 					obj->fd_array_cnt + 1);  in bpf_object__resolve_ksym_func_btf_id()
8225 		mod_btf->fd_array_idx = obj->fd_array_cnt;  in bpf_object__resolve_ksym_func_btf_id()
8227 		obj->fd_array[obj->fd_array_cnt++] = mod_btf->fd;  in bpf_object__resolve_ksym_func_btf_id()
8230 	ext->is_set = true;  in bpf_object__resolve_ksym_func_btf_id()
8231 	ext->ksym.kernel_btf_id = kfunc_id;  in bpf_object__resolve_ksym_func_btf_id()
8232 	ext->ksym.btf_fd_idx = mod_btf ? mod_btf->fd_array_idx : 0;  in bpf_object__resolve_ksym_func_btf_id()
8235 	 * {kernel_btf_id, btf_fd_idx} -> fixup bpf_call.  in bpf_object__resolve_ksym_func_btf_id()
8236 	 * {kernel_btf_id, kernel_btf_obj_fd} -> fixup ld_imm64.  in bpf_object__resolve_ksym_func_btf_id()
8238 	ext->ksym.kernel_btf_obj_fd = mod_btf ? mod_btf->fd : 0;  in bpf_object__resolve_ksym_func_btf_id()
8240 		 ext->name, mod_btf ? mod_btf->name : "vmlinux", kfunc_id);  in bpf_object__resolve_ksym_func_btf_id()
8248 	struct extern_desc *ext;  in bpf_object__resolve_ksyms_btf_id()  local
8251 	for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__resolve_ksyms_btf_id()
8252 		ext = &obj->externs[i];  in bpf_object__resolve_ksyms_btf_id()
8253 		if (ext->type != EXT_KSYM || !ext->ksym.type_id)  in bpf_object__resolve_ksyms_btf_id()
8256 		if (obj->gen_loader) {  in bpf_object__resolve_ksyms_btf_id()
8257 			ext->is_set = true;  in bpf_object__resolve_ksyms_btf_id()
8258 			ext->ksym.kernel_btf_obj_fd = 0;  in bpf_object__resolve_ksyms_btf_id()
8259 			ext->ksym.kernel_btf_id = 0;  in bpf_object__resolve_ksyms_btf_id()
8262 		t = btf__type_by_id(obj->btf, ext->btf_id);  in bpf_object__resolve_ksyms_btf_id()
8264 			err = bpf_object__resolve_ksym_var_btf_id(obj, ext);  in bpf_object__resolve_ksyms_btf_id()
8266 			err = bpf_object__resolve_ksym_func_btf_id(obj, ext);  in bpf_object__resolve_ksyms_btf_id()
8278 	struct extern_desc *ext;  in bpf_object__resolve_externs()  local
8282 	if (obj->nr_extern == 0)  in bpf_object__resolve_externs()
8285 	if (obj->kconfig_map_idx >= 0)  in bpf_object__resolve_externs()
8286 		kcfg_data = obj->maps[obj->kconfig_map_idx].mmaped;  in bpf_object__resolve_externs()
8288 	for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__resolve_externs()
8289 		ext = &obj->externs[i];  in bpf_object__resolve_externs()
8291 		if (ext->type == EXT_KSYM) {  in bpf_object__resolve_externs()
8292 			if (ext->ksym.type_id)  in bpf_object__resolve_externs()
8297 		} else if (ext->type == EXT_KCFG) {  in bpf_object__resolve_externs()
8298 			void *ext_ptr = kcfg_data + ext->kcfg.data_off;  in bpf_object__resolve_externs()
8301 			/* Kconfig externs need actual /proc/config.gz */  in bpf_object__resolve_externs()
8302 			if (str_has_pfx(ext->name, "CONFIG_")) {  in bpf_object__resolve_externs()
8308 			if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) {  in bpf_object__resolve_externs()
8311 					pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name);  in bpf_object__resolve_externs()
8312 					return -EINVAL;  in bpf_object__resolve_externs()
8314 			} else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) {  in bpf_object__resolve_externs()
8316 			} else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) {  in bpf_object__resolve_externs()
8318 			} else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) {  in bpf_object__resolve_externs()
8321 				 * customly by libbpf (their values don't come from Kconfig).  in bpf_object__resolve_externs()
8326 				pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name);  in bpf_object__resolve_externs()
8327 				return -EINVAL;  in bpf_object__resolve_externs()
8330 			err = set_kcfg_value_num(ext, ext_ptr, value);  in bpf_object__resolve_externs()
8334 				 ext->name, (long long)value);  in bpf_object__resolve_externs()
8336 			pr_warn("extern '%s': unrecognized extern kind\n", ext->name);  in bpf_object__resolve_externs()
8337 			return -EINVAL;  in bpf_object__resolve_externs()
8343 			return -EINVAL;  in bpf_object__resolve_externs()
8345 		for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__resolve_externs()
8346 			ext = &obj->externs[i];  in bpf_object__resolve_externs()
8347 			if (ext->type == EXT_KCFG && !ext->is_set) {  in bpf_object__resolve_externs()
8356 			return -EINVAL;  in bpf_object__resolve_externs()
8361 			return -EINVAL;  in bpf_object__resolve_externs()
8366 			return -EINVAL;  in bpf_object__resolve_externs()
8368 	for (i = 0; i < obj->nr_extern; i++) {  in bpf_object__resolve_externs()
8369 		ext = &obj->externs[i];  in bpf_object__resolve_externs()
8371 		if (!ext->is_set && !ext->is_weak) {  in bpf_object__resolve_externs()
8372 			pr_warn("extern '%s' (strong): not resolved\n", ext->name);  in bpf_object__resolve_externs()
8373 			return -ESRCH;  in bpf_object__resolve_externs()
8374 		} else if (!ext->is_set) {  in bpf_object__resolve_externs()
8376 				 ext->name);  in bpf_object__resolve_externs()
8388 	st_ops = map->st_ops;  in bpf_map_prepare_vdata()
8389 	for (i = 0; i < btf_vlen(st_ops->type); i++) {  in bpf_map_prepare_vdata()
8390 		struct bpf_program *prog = st_ops->progs[i];  in bpf_map_prepare_vdata()
8398 		kern_data = st_ops->kern_vdata + st_ops->kern_func_off[i];  in bpf_map_prepare_vdata()
8408 	for (i = 0; i < obj->nr_maps; i++) {  in bpf_object_prepare_struct_ops()
8409 		map = &obj->maps[i];  in bpf_object_prepare_struct_ops()
8414 		if (!map->autocreate)  in bpf_object_prepare_struct_ops()
8428 		return libbpf_err(-EINVAL);  in bpf_object_load()
8430 	if (obj->loaded) {  in bpf_object_load()
8431 		pr_warn("object '%s': load can't be attempted twice\n", obj->name);  in bpf_object_load()
8432 		return libbpf_err(-EINVAL);  in bpf_object_load()
8435 	if (obj->gen_loader)  in bpf_object_load()
8436 		bpf_gen__init(obj->gen_loader, extra_log_level, obj->nr_programs, obj->nr_maps);  in bpf_object_load()
8441 	err = err ? : bpf_object__resolve_externs(obj, obj->kconfig);  in bpf_object_load()
8445 	err = err ? : bpf_object__relocate(obj, obj->btf_custom_path ? : target_btf_path);  in bpf_object_load()
8452 	if (obj->gen_loader) {  in bpf_object_load()
8454 		if (obj->btf)  in bpf_object_load()
8455 			btf__set_fd(obj->btf, -1);  in bpf_object_load()
8457 			err = bpf_gen__finish(obj->gen_loader, obj->nr_programs, obj->nr_maps);  in bpf_object_load()
8461 	zfree(&obj->fd_array);  in bpf_object_load()
8464 	for (i = 0; i < obj->btf_module_cnt; i++) {  in bpf_object_load()
8465 		close(obj->btf_modules[i].fd);  in bpf_object_load()
8466 		btf__free(obj->btf_modules[i].btf);  in bpf_object_load()
8467 		free(obj->btf_modules[i].name);  in bpf_object_load()
8469 	free(obj->btf_modules);  in bpf_object_load()
8472 	btf__free(obj->btf_vmlinux);  in bpf_object_load()
8473 	obj->btf_vmlinux = NULL;  in bpf_object_load()
8475 	obj->loaded = true; /* doesn't matter if successfully or not */  in bpf_object_load()
8482 	/* unpin any maps that were auto-pinned during load */  in bpf_object_load()
8483 	for (i = 0; i < obj->nr_maps; i++)  in bpf_object_load()
8484 		if (obj->maps[i].pinned && !obj->maps[i].reused)  in bpf_object_load()
8485 			bpf_map__unpin(&obj->maps[i], NULL);  in bpf_object_load()
8488 	pr_warn("failed to load object '%s'\n", obj->path);  in bpf_object_load()
8505 		return -ENOMEM;  in make_parent_dir()
8509 		err = -errno;  in make_parent_dir()
8513 		cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));  in make_parent_dir()
8527 		return -EINVAL;  in check_path()
8531 		return -ENOMEM;  in check_path()
8537 		err = -errno;  in check_path()
8543 		err = -EINVAL;  in check_path()
8554 	if (prog->fd < 0) {  in bpf_program__pin()
8555 		pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name);  in bpf_program__pin()
8556 		return libbpf_err(-EINVAL);  in bpf_program__pin()
8567 	if (bpf_obj_pin(prog->fd, path)) {  in bpf_program__pin()
8568 		err = -errno;  in bpf_program__pin()
8570 		pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp);  in bpf_program__pin()
8574 	pr_debug("prog '%s': pinned at '%s'\n", prog->name, path);  in bpf_program__pin()
8582 	if (prog->fd < 0) {  in bpf_program__unpin()
8583 		pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name);  in bpf_program__unpin()
8584 		return libbpf_err(-EINVAL);  in bpf_program__unpin()
8593 		return libbpf_err(-errno);  in bpf_program__unpin()
8595 	pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path);  in bpf_program__unpin()
8606 		return libbpf_err(-EINVAL);  in bpf_map__pin()
8609 	if (map->fd < 0) {  in bpf_map__pin()
8610 		pr_warn("map '%s': can't pin BPF map without FD (was it created?)\n", map->name);  in bpf_map__pin()
8611 		return libbpf_err(-EINVAL);  in bpf_map__pin()
8614 	if (map->pin_path) {  in bpf_map__pin()
8615 		if (path && strcmp(path, map->pin_path)) {  in bpf_map__pin()
8617 				bpf_map__name(map), map->pin_path, path);  in bpf_map__pin()
8618 			return libbpf_err(-EINVAL);  in bpf_map__pin()
8619 		} else if (map->pinned) {  in bpf_map__pin()
8620 			pr_debug("map '%s' already pinned at '%s'; not re-pinning\n",  in bpf_map__pin()
8621 				 bpf_map__name(map), map->pin_path);  in bpf_map__pin()
8628 			return libbpf_err(-EINVAL);  in bpf_map__pin()
8629 		} else if (map->pinned) {  in bpf_map__pin()
8631 			return libbpf_err(-EEXIST);  in bpf_map__pin()
8634 		map->pin_path = strdup(path);  in bpf_map__pin()
8635 		if (!map->pin_path) {  in bpf_map__pin()
8636 			err = -errno;  in bpf_map__pin()
8641 	err = make_parent_dir(map->pin_path);  in bpf_map__pin()
8645 	err = check_path(map->pin_path);  in bpf_map__pin()
8649 	if (bpf_obj_pin(map->fd, map->pin_path)) {  in bpf_map__pin()
8650 		err = -errno;  in bpf_map__pin()
8654 	map->pinned = true;  in bpf_map__pin()
8655 	pr_debug("pinned map '%s'\n", map->pin_path);  in bpf_map__pin()
8660 	cp = libbpf_strerror_r(-err, errmsg, sizeof(errmsg));  in bpf_map__pin()
8671 		return libbpf_err(-EINVAL);  in bpf_map__unpin()
8674 	if (map->pin_path) {  in bpf_map__unpin()
8675 		if (path && strcmp(path, map->pin_path)) {  in bpf_map__unpin()
8677 				bpf_map__name(map), map->pin_path, path);  in bpf_map__unpin()
8678 			return libbpf_err(-EINVAL);  in bpf_map__unpin()
8680 		path = map->pin_path;  in bpf_map__unpin()
8684 		return libbpf_err(-EINVAL);  in bpf_map__unpin()
8693 		return libbpf_err(-errno);  in bpf_map__unpin()
8695 	map->pinned = false;  in bpf_map__unpin()
8708 			return libbpf_err(-errno);  in bpf_map__set_pin_path()
8711 	free(map->pin_path);  in bpf_map__set_pin_path()
8712 	map->pin_path = new;  in bpf_map__set_pin_path()
8721 	return map->pin_path;  in bpf_map__pin_path()
8726 	return map->pinned;  in bpf_map__is_pinned()
8745 		return libbpf_err(-ENOENT);  in bpf_object__pin_maps()
8747 	if (!obj->loaded) {  in bpf_object__pin_maps()
8749 		return libbpf_err(-ENOENT);  in bpf_object__pin_maps()
8756 		if (!map->autocreate)  in bpf_object__pin_maps()
8765 		} else if (!map->pin_path) {  in bpf_object__pin_maps()
8778 		if (!map->pin_path)  in bpf_object__pin_maps()
8793 		return libbpf_err(-ENOENT);  in bpf_object__unpin_maps()
8805 		} else if (!map->pin_path) {  in bpf_object__unpin_maps()
8824 		return libbpf_err(-ENOENT);  in bpf_object__pin_programs()
8826 	if (!obj->loaded) {  in bpf_object__pin_programs()
8828 		return libbpf_err(-ENOENT);  in bpf_object__pin_programs()
8832 		err = pathname_concat(buf, sizeof(buf), path, prog->name);  in bpf_object__pin_programs()
8845 		if (pathname_concat(buf, sizeof(buf), path, prog->name))  in bpf_object__pin_programs()
8860 		return libbpf_err(-ENOENT);  in bpf_object__unpin_programs()
8865 		err = pathname_concat(buf, sizeof(buf), path, prog->name);  in bpf_object__unpin_programs()
8911 	if (map->inner_map) {  in bpf_map__destroy()
8912 		bpf_map__destroy(map->inner_map);  in bpf_map__destroy()
8913 		zfree(&map->inner_map);  in bpf_map__destroy()
8916 	zfree(&map->init_slots);  in bpf_map__destroy()
8917 	map->init_slots_sz = 0;  in bpf_map__destroy()
8919 	if (map->mmaped && map->mmaped != map->obj->arena_data)  in bpf_map__destroy()
8920 		munmap(map->mmaped, bpf_map_mmap_sz(map));  in bpf_map__destroy()
8921 	map->mmaped = NULL;  in bpf_map__destroy()
8923 	if (map->st_ops) {  in bpf_map__destroy()
8924 		zfree(&map->st_ops->data);  in bpf_map__destroy()
8925 		zfree(&map->st_ops->progs);  in bpf_map__destroy()
8926 		zfree(&map->st_ops->kern_func_off);  in bpf_map__destroy()
8927 		zfree(&map->st_ops);  in bpf_map__destroy()
8930 	zfree(&map->name);  in bpf_map__destroy()
8931 	zfree(&map->real_name);  in bpf_map__destroy()
8932 	zfree(&map->pin_path);  in bpf_map__destroy()
8934 	if (map->fd >= 0)  in bpf_map__destroy()
8935 		zclose(map->fd);  in bpf_map__destroy()
8945 	usdt_manager_free(obj->usdt_man);  in bpf_object__close()
8946 	obj->usdt_man = NULL;  in bpf_object__close()
8948 	bpf_gen__free(obj->gen_loader);  in bpf_object__close()
8951 	btf__free(obj->btf);  in bpf_object__close()
8952 	btf__free(obj->btf_vmlinux);  in bpf_object__close()
8953 	btf_ext__free(obj->btf_ext);  in bpf_object__close()
8955 	for (i = 0; i < obj->nr_maps; i++)  in bpf_object__close()
8956 		bpf_map__destroy(&obj->maps[i]);  in bpf_object__close()
8958 	zfree(&obj->btf_custom_path);  in bpf_object__close()
8959 	zfree(&obj->kconfig);  in bpf_object__close()
8961 	for (i = 0; i < obj->nr_extern; i++)  in bpf_object__close()
8962 		zfree(&obj->externs[i].essent_name);  in bpf_object__close()
8964 	zfree(&obj->externs);  in bpf_object__close()
8965 	obj->nr_extern = 0;  in bpf_object__close()
8967 	zfree(&obj->maps);  in bpf_object__close()
8968 	obj->nr_maps = 0;  in bpf_object__close()
8970 	if (obj->programs && obj->nr_programs) {  in bpf_object__close()
8971 		for (i = 0; i < obj->nr_programs; i++)  in bpf_object__close()
8972 			bpf_program__exit(&obj->programs[i]);  in bpf_object__close()
8974 	zfree(&obj->programs);  in bpf_object__close()
8976 	zfree(&obj->feat_cache);  in bpf_object__close()
8977 	zfree(&obj->token_path);  in bpf_object__close()
8978 	if (obj->token_fd > 0)  in bpf_object__close()
8979 		close(obj->token_fd);  in bpf_object__close()
8981 	zfree(&obj->arena_data);  in bpf_object__close()
8988 	return obj ? obj->name : libbpf_err_ptr(-EINVAL);  in bpf_object__name()
8993 	return obj ? obj->kern_version : 0;  in bpf_object__kversion()
8998 	return obj ? obj->btf : NULL;  in bpf_object__btf()
9003 	return obj->btf ? btf__fd(obj->btf) : -1;  in bpf_object__btf_fd()
9008 	if (obj->loaded)  in bpf_object__set_kversion()
9009 		return libbpf_err(-EINVAL);  in bpf_object__set_kversion()
9011 	obj->kern_version = kern_version;  in bpf_object__set_kversion()
9021 		return -EFAULT;  in bpf_object__gen_loader()
9023 		return -EINVAL;  in bpf_object__gen_loader()
9026 		return -ENOMEM;  in bpf_object__gen_loader()
9027 	gen->opts = opts;  in bpf_object__gen_loader()
9028 	obj->gen_loader = gen;  in bpf_object__gen_loader()
9036 	size_t nr_programs = obj->nr_programs;  in __bpf_program__iter()
9044 		return forward ? &obj->programs[0] :  in __bpf_program__iter()
9045 			&obj->programs[nr_programs - 1];  in __bpf_program__iter()
9047 	if (p->obj != obj) {  in __bpf_program__iter()
9052 	idx = (p - obj->programs) + (forward ? 1 : -1);  in __bpf_program__iter()
9053 	if (idx >= obj->nr_programs || idx < 0)  in __bpf_program__iter()
9055 	return &obj->programs[idx];  in __bpf_program__iter()
9084 	prog->prog_ifindex = ifindex;  in bpf_program__set_ifindex()
9089 	return prog->name;  in bpf_program__name()
9094 	return prog->sec_name;  in bpf_program__section_name()
9099 	return prog->autoload;  in bpf_program__autoload()
9104 	if (prog->obj->loaded)  in bpf_program__set_autoload()
9105 		return libbpf_err(-EINVAL);  in bpf_program__set_autoload()
9107 	prog->autoload = autoload;  in bpf_program__set_autoload()
9113 	return prog->autoattach;  in bpf_program__autoattach()
9118 	prog->autoattach = autoattach;  in bpf_program__set_autoattach()
9123 	return prog->insns;  in bpf_program__insns()
9128 	return prog->insns_cnt;  in bpf_program__insn_cnt()
9136 	if (prog->obj->loaded)  in bpf_program__set_insns()
9137 		return -EBUSY;  in bpf_program__set_insns()
9139 	insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns));  in bpf_program__set_insns()
9142 		pr_warn("prog '%s': failed to realloc prog code\n", prog->name);  in bpf_program__set_insns()
9143 		return -ENOMEM;  in bpf_program__set_insns()
9147 	prog->insns = insns;  in bpf_program__set_insns()
9148 	prog->insns_cnt = new_insn_cnt;  in bpf_program__set_insns()
9155 		return libbpf_err(-EINVAL);  in bpf_program__fd()
9157 	if (prog->fd < 0)  in bpf_program__fd()
9158 		return libbpf_err(-ENOENT);  in bpf_program__fd()
9160 	return prog->fd;  in bpf_program__fd()
9168 	return prog->type;  in bpf_program__type()
9179 	if (prog->obj->loaded)  in bpf_program__set_type()
9180 		return libbpf_err(-EBUSY);  in bpf_program__set_type()
9183 	if (prog->type == type)  in bpf_program__set_type()
9186 	prog->type = type;  in bpf_program__set_type()
9190 	 * fallback handler, which by definition is program type-agnostic and  in bpf_program__set_type()
9191 	 * is a catch-all custom handler, optionally set by the application,  in bpf_program__set_type()
9194 	if (prog->sec_def != &custom_fallback_def)  in bpf_program__set_type()
9195 		prog->sec_def = NULL;  in bpf_program__set_type()
9204 	return prog->expected_attach_type;  in bpf_program__expected_attach_type()
9210 	if (prog->obj->loaded)  in bpf_program__set_expected_attach_type()
9211 		return libbpf_err(-EBUSY);  in bpf_program__set_expected_attach_type()
9213 	prog->expected_attach_type = type;  in bpf_program__set_expected_attach_type()
9219 	return prog->prog_flags;  in bpf_program__flags()
9224 	if (prog->obj->loaded)  in bpf_program__set_flags()
9225 		return libbpf_err(-EBUSY);  in bpf_program__set_flags()
9227 	prog->prog_flags = flags;  in bpf_program__set_flags()
9233 	return prog->log_level;  in bpf_program__log_level()
9238 	if (prog->obj->loaded)  in bpf_program__set_log_level()
9239 		return libbpf_err(-EBUSY);  in bpf_program__set_log_level()
9241 	prog->log_level = log_level;  in bpf_program__set_log_level()
9247 	*log_size = prog->log_size;  in bpf_program__log_buf()
9248 	return prog->log_buf;  in bpf_program__log_buf()
9254 		return -EINVAL;  in bpf_program__set_log_buf()
9255 	if (prog->log_size > UINT_MAX)  in bpf_program__set_log_buf()
9256 		return -EINVAL;  in bpf_program__set_log_buf()
9257 	if (prog->obj->loaded)  in bpf_program__set_log_buf()
9258 		return -EBUSY;  in bpf_program__set_log_buf()
9260 	prog->log_buf = log_buf;  in bpf_program__set_log_buf()
9261 	prog->log_size = log_size;  in bpf_program__set_log_buf()
9328 	SEC_DEF("freplace+",		EXT, 0, SEC_ATTACH_BTF, attach_trace),
9397 		return libbpf_err(-EINVAL);  in libbpf_register_prog_handler()
9400 		return libbpf_err(-E2BIG);  in libbpf_register_prog_handler()
9406 			return libbpf_err(-ENOMEM);  in libbpf_register_prog_handler()
9412 			return libbpf_err(-EBUSY);  in libbpf_register_prog_handler()
9417 	sec_def->sec = sec ? strdup(sec) : NULL;  in libbpf_register_prog_handler()
9418 	if (sec && !sec_def->sec)  in libbpf_register_prog_handler()
9419 		return libbpf_err(-ENOMEM);  in libbpf_register_prog_handler()
9421 	sec_def->prog_type = prog_type;  in libbpf_register_prog_handler()
9422 	sec_def->expected_attach_type = exp_attach_type;  in libbpf_register_prog_handler()
9423 	sec_def->cookie = OPTS_GET(opts, cookie, 0);  in libbpf_register_prog_handler()
9425 	sec_def->prog_setup_fn = OPTS_GET(opts, prog_setup_fn, NULL);  in libbpf_register_prog_handler()
9426 	sec_def->prog_prepare_load_fn = OPTS_GET(opts, prog_prepare_load_fn, NULL);  in libbpf_register_prog_handler()
9427 	sec_def->prog_attach_fn = OPTS_GET(opts, prog_attach_fn, NULL);  in libbpf_register_prog_handler()
9429 	sec_def->handler_id = ++last_custom_sec_def_handler_id;  in libbpf_register_prog_handler()
9436 	return sec_def->handler_id;  in libbpf_register_prog_handler()
9445 		return libbpf_err(-EINVAL);  in libbpf_unregister_prog_handler()
9459 		return libbpf_err(-ENOENT);  in libbpf_unregister_prog_handler()
9463 		custom_sec_defs[i - 1] = custom_sec_defs[i];  in libbpf_unregister_prog_handler()
9464 	custom_sec_def_cnt--;  in libbpf_unregister_prog_handler()
9480 	size_t len = strlen(sec_def->sec);  in sec_def_matches()
9483 	if (sec_def->sec[len - 1] == '/') {  in sec_def_matches()
9484 		if (str_has_pfx(sec_name, sec_def->sec))  in sec_def_matches()
9490 	 * well-formed SEC("type/extras") with proper '/' separator  in sec_def_matches()
9492 	if (sec_def->sec[len - 1] == '+') {  in sec_def_matches()
9493 		len--;  in sec_def_matches()
9495 		if (strncmp(sec_name, sec_def->sec, len) != 0)  in sec_def_matches()
9503 	return strcmp(sec_name, sec_def->sec) == 0;  in sec_def_matches()
9548 			if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)  in libbpf_get_type_names()
9551 			if (!(sec_def->cookie & SEC_ATTACHABLE))  in libbpf_get_type_names()
9573 		return libbpf_err(-EINVAL);  in libbpf_prog_type_by_name()
9577 		*prog_type = sec_def->prog_type;  in libbpf_prog_type_by_name()
9578 		*expected_attach_type = sec_def->expected_attach_type;  in libbpf_prog_type_by_name()
9589 	return libbpf_err(-ESRCH);  in libbpf_prog_type_by_name()
9631 	for (i = 0; i < obj->nr_maps; i++) {  in find_struct_ops_map_by_offset()
9632 		map = &obj->maps[i];  in find_struct_ops_map_by_offset()
9635 		if (map->sec_idx == sec_idx &&  in find_struct_ops_map_by_offset()
9636 		    map->sec_offset <= offset &&  in find_struct_ops_map_by_offset()
9637 		    offset - map->sec_offset < map->def.value_size)  in find_struct_ops_map_by_offset()
9644 /* Collect the reloc from ELF, populate the st_ops->progs[], and update
9645  * st_ops->data for shadow type.
9663 	btf = obj->btf;  in bpf_object__collect_st_ops_relos()
9664 	nrels = shdr->sh_size / shdr->sh_entsize;  in bpf_object__collect_st_ops_relos()
9669 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_st_ops_relos()
9672 		sym = elf_sym_by_idx(obj, ELF64_R_SYM(rel->r_info));  in bpf_object__collect_st_ops_relos()
9675 				(size_t)ELF64_R_SYM(rel->r_info));  in bpf_object__collect_st_ops_relos()
9676 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_st_ops_relos()
9679 		name = elf_sym_str(obj, sym->st_name) ?: "<?>";  in bpf_object__collect_st_ops_relos()
9680 		map = find_struct_ops_map_by_offset(obj, shdr->sh_info, rel->r_offset);  in bpf_object__collect_st_ops_relos()
9682 			pr_warn("struct_ops reloc: cannot find map at rel->r_offset %zu\n",  in bpf_object__collect_st_ops_relos()
9683 				(size_t)rel->r_offset);  in bpf_object__collect_st_ops_relos()
9684 			return -EINVAL;  in bpf_object__collect_st_ops_relos()
9687 		moff = rel->r_offset - map->sec_offset;  in bpf_object__collect_st_ops_relos()
9688 		shdr_idx = sym->st_shndx;  in bpf_object__collect_st_ops_relos()
9689 		st_ops = map->st_ops;  in bpf_object__collect_st_ops_relos()
9690 …pr_debug("struct_ops reloc %s: for %lld value %lld shdr_idx %u rel->r_offset %zu map->sec_offset %…  in bpf_object__collect_st_ops_relos()
9691 			 map->name,  in bpf_object__collect_st_ops_relos()
9692 			 (long long)(rel->r_info >> 32),  in bpf_object__collect_st_ops_relos()
9693 			 (long long)sym->st_value,  in bpf_object__collect_st_ops_relos()
9694 			 shdr_idx, (size_t)rel->r_offset,  in bpf_object__collect_st_ops_relos()
9695 			 map->sec_offset, sym->st_name, name);  in bpf_object__collect_st_ops_relos()
9698 			pr_warn("struct_ops reloc %s: rel->r_offset %zu shdr_idx %u unsupported non-static function\n",  in bpf_object__collect_st_ops_relos()
9699 				map->name, (size_t)rel->r_offset, shdr_idx);  in bpf_object__collect_st_ops_relos()
9700 			return -LIBBPF_ERRNO__RELOC;  in bpf_object__collect_st_ops_relos()
9702 		if (sym->st_value % BPF_INSN_SZ) {  in bpf_object__collect_st_ops_relos()
9704 				map->name, (unsigned long long)sym->st_value);  in bpf_object__collect_st_ops_relos()
9705 			return -LIBBPF_ERRNO__FORMAT;  in bpf_object__collect_st_ops_relos()
9707 		insn_idx = sym->st_value / BPF_INSN_SZ;  in bpf_object__collect_st_ops_relos()
9709 		member = find_member_by_offset(st_ops->type, moff * 8);  in bpf_object__collect_st_ops_relos()
9712 				map->name, moff);  in bpf_object__collect_st_ops_relos()
9713 			return -EINVAL;  in bpf_object__collect_st_ops_relos()
9715 		member_idx = member - btf_members(st_ops->type);  in bpf_object__collect_st_ops_relos()
9716 		name = btf__name_by_offset(btf, member->name_off);  in bpf_object__collect_st_ops_relos()
9718 		if (!resolve_func_ptr(btf, member->type, NULL)) {  in bpf_object__collect_st_ops_relos()
9720 				map->name, name);  in bpf_object__collect_st_ops_relos()
9721 			return -EINVAL;  in bpf_object__collect_st_ops_relos()
9727 				map->name, shdr_idx, name);  in bpf_object__collect_st_ops_relos()
9728 			return -EINVAL;  in bpf_object__collect_st_ops_relos()
9732 		if (prog->type != BPF_PROG_TYPE_STRUCT_OPS) {  in bpf_object__collect_st_ops_relos()
9734 				map->name, prog->name);  in bpf_object__collect_st_ops_relos()
9735 			return -EINVAL;  in bpf_object__collect_st_ops_relos()
9738 		st_ops->progs[member_idx] = prog;  in bpf_object__collect_st_ops_relos()
9740 		/* st_ops->data will be exposed to users, being returned by  in bpf_object__collect_st_ops_relos()
9746 		*((struct bpf_program **)(st_ops->data + moff)) = prog;  in bpf_object__collect_st_ops_relos()
9793 		return -ENAMETOOLONG;  in find_btf_by_prefix_kind()
9843 	err = -EINVAL;  in libbpf_find_prog_btf_id()
9870 	ret = find_attach_btf_id(obj->btf_vmlinux, attach_name, attach_type);  in find_kernel_btf_id()
9876 	if (ret != -ENOENT)  in find_kernel_btf_id()
9883 	for (i = 0; i < obj->btf_module_cnt; i++) {  in find_kernel_btf_id()
9884 		const struct module_btf *mod = &obj->btf_modules[i];  in find_kernel_btf_id()
9886 		ret = find_attach_btf_id(mod->btf, attach_name, attach_type);  in find_kernel_btf_id()
9888 			*btf_obj_fd = mod->fd;  in find_kernel_btf_id()
9892 		if (ret == -ENOENT)  in find_kernel_btf_id()
9898 	return -ESRCH;  in find_kernel_btf_id()
9904 	enum bpf_attach_type attach_type = prog->expected_attach_type;  in libbpf_find_attach_btf_id()
9905 	__u32 attach_prog_fd = prog->attach_prog_fd;  in libbpf_find_attach_btf_id()
9909 	if (prog->type == BPF_PROG_TYPE_EXT || attach_prog_fd) {  in libbpf_find_attach_btf_id()
9911 			pr_warn("prog '%s': attach program FD is not set\n", prog->name);  in libbpf_find_attach_btf_id()
9912 			return -EINVAL;  in libbpf_find_attach_btf_id()
9917 				 prog->name, attach_prog_fd, attach_name, err);  in libbpf_find_attach_btf_id()
9926 	if (prog->obj->gen_loader) {  in libbpf_find_attach_btf_id()
9927 		bpf_gen__record_attach_target(prog->obj->gen_loader, attach_name, attach_type);  in libbpf_find_attach_btf_id()
9931 		err = find_kernel_btf_id(prog->obj, attach_name,  in libbpf_find_attach_btf_id()
9937 			prog->name, attach_name, err);  in libbpf_find_attach_btf_id()
9950 		return libbpf_err(-EINVAL);  in libbpf_attach_type_by_name()
9961 		return libbpf_err(-EINVAL);  in libbpf_attach_type_by_name()
9964 	if (sec_def->prog_prepare_load_fn != libbpf_prepare_prog_load)  in libbpf_attach_type_by_name()
9965 		return libbpf_err(-EINVAL);  in libbpf_attach_type_by_name()
9966 	if (!(sec_def->cookie & SEC_ATTACHABLE))  in libbpf_attach_type_by_name()
9967 		return libbpf_err(-EINVAL);  in libbpf_attach_type_by_name()
9969 	*attach_type = sec_def->expected_attach_type;  in libbpf_attach_type_by_name()
9976 		return libbpf_err(-EINVAL);  in bpf_map__fd()
9978 		return -1;  in bpf_map__fd()
9979 	return map->fd;  in bpf_map__fd()
9985 	 * their user-visible name differs from kernel-visible name. Users see  in map_uses_real_name()
9990 	if (map->libbpf_type == LIBBPF_MAP_DATA && strcmp(map->real_name, DATA_SEC) != 0)  in map_uses_real_name()
9992 	if (map->libbpf_type == LIBBPF_MAP_RODATA && strcmp(map->real_name, RODATA_SEC) != 0)  in map_uses_real_name()
10003 		return map->real_name;  in bpf_map__name()
10005 	return map->name;  in bpf_map__name()
10010 	return map->def.type;  in bpf_map__type()
10016 		return libbpf_err(-EBUSY);  in bpf_map__set_type()
10017 	map->def.type = type;  in bpf_map__set_type()
10023 	return map->def.map_flags;  in bpf_map__map_flags()
10029 		return libbpf_err(-EBUSY);  in bpf_map__set_map_flags()
10030 	map->def.map_flags = flags;  in bpf_map__set_map_flags()
10036 	return map->map_extra;  in bpf_map__map_extra()
10042 		return libbpf_err(-EBUSY);  in bpf_map__set_map_extra()
10043 	map->map_extra = map_extra;  in bpf_map__set_map_extra()
10049 	return map->numa_node;  in bpf_map__numa_node()
10055 		return libbpf_err(-EBUSY);  in bpf_map__set_numa_node()
10056 	map->numa_node = numa_node;  in bpf_map__set_numa_node()
10062 	return map->def.key_size;  in bpf_map__key_size()
10068 		return libbpf_err(-EBUSY);  in bpf_map__set_key_size()
10069 	map->def.key_size = size;  in bpf_map__set_key_size()
10075 	return map->def.value_size;  in bpf_map__value_size()
10089 	btf = bpf_object__btf(map->obj);  in map_btf_datasec_resize()
10091 		return -ENOENT;  in map_btf_datasec_resize()
10098 		return -EINVAL;  in map_btf_datasec_resize()
10106 		return -EINVAL;  in map_btf_datasec_resize()
10110 	var = &btf_var_secinfos(datasec_type)[vlen - 1];  in map_btf_datasec_resize()
10111 	var_type = btf_type_by_id(btf, var->type);  in map_btf_datasec_resize()
10112 	array_type = skip_mods_and_typedefs(btf, var_type->type, NULL);  in map_btf_datasec_resize()
10116 		return -EINVAL;  in map_btf_datasec_resize()
10121 	element_sz = btf__resolve_size(btf, array->type);  in map_btf_datasec_resize()
10122 	if (element_sz <= 0 || (size - var->offset) % element_sz != 0) {  in map_btf_datasec_resize()
10125 		return -EINVAL;  in map_btf_datasec_resize()
10129 	nr_elements = (size - var->offset) / element_sz;  in map_btf_datasec_resize()
10130 	new_array_id = btf__add_array(btf, array->index_type, array->type, nr_elements);  in map_btf_datasec_resize()
10137 	datasec_type = btf_type_by_id(btf, map->btf_value_type_id);  in map_btf_datasec_resize()
10138 	var = &btf_var_secinfos(datasec_type)[vlen - 1];  in map_btf_datasec_resize()
10139 	var_type = btf_type_by_id(btf, var->type);  in map_btf_datasec_resize()
10142 	datasec_type->size = size;  in map_btf_datasec_resize()
10143 	var->size = size - var->offset;  in map_btf_datasec_resize()
10144 	var_type->type = new_array_id;  in map_btf_datasec_resize()
10151 	if (map->obj->loaded || map->reused)  in bpf_map__set_value_size()
10152 		return libbpf_err(-EBUSY);  in bpf_map__set_value_size()
10154 	if (map->mmaped) {  in bpf_map__set_value_size()
10158 		if (map->def.type != BPF_MAP_TYPE_ARRAY)  in bpf_map__set_value_size()
10159 			return -EOPNOTSUPP;  in bpf_map__set_value_size()
10162 		mmap_new_sz = array_map_mmap_sz(size, map->def.max_entries);  in bpf_map__set_value_size()
10165 			pr_warn("map '%s': failed to resize memory-mapped region: %d\n",  in bpf_map__set_value_size()
10170 		if (err && err != -ENOENT) {  in bpf_map__set_value_size()
10173 			map->btf_value_type_id = 0;  in bpf_map__set_value_size()
10174 			map->btf_key_type_id = 0;  in bpf_map__set_value_size()
10178 	map->def.value_size = size;  in bpf_map__set_value_size()
10184 	return map ? map->btf_key_type_id : 0;  in bpf_map__btf_key_type_id()
10189 	return map ? map->btf_value_type_id : 0;  in bpf_map__btf_value_type_id()
10197 	if (map->obj->loaded || map->reused)  in bpf_map__set_initial_value()
10198 		return libbpf_err(-EBUSY);  in bpf_map__set_initial_value()
10200 	if (!map->mmaped || map->libbpf_type == LIBBPF_MAP_KCONFIG)  in bpf_map__set_initial_value()
10201 		return libbpf_err(-EINVAL);  in bpf_map__set_initial_value()
10203 	if (map->def.type == BPF_MAP_TYPE_ARENA)  in bpf_map__set_initial_value()
10204 		actual_sz = map->obj->arena_data_sz;  in bpf_map__set_initial_value()
10206 		actual_sz = map->def.value_size;  in bpf_map__set_initial_value()
10208 		return libbpf_err(-EINVAL);  in bpf_map__set_initial_value()
10210 	memcpy(map->mmaped, data, size);  in bpf_map__set_initial_value()
10218 			*psize = map->def.value_size;  in bpf_map__initial_value()
10219 		return map->st_ops->data;  in bpf_map__initial_value()
10222 	if (!map->mmaped)  in bpf_map__initial_value()
10225 	if (map->def.type == BPF_MAP_TYPE_ARENA)  in bpf_map__initial_value()
10226 		*psize = map->obj->arena_data_sz;  in bpf_map__initial_value()
10228 		*psize = map->def.value_size;  in bpf_map__initial_value()
10230 	return map->mmaped;  in bpf_map__initial_value()
10235 	return map->libbpf_type != LIBBPF_MAP_UNSPEC;  in bpf_map__is_internal()
10240 	return map->map_ifindex;  in bpf_map__ifindex()
10246 		return libbpf_err(-EBUSY);  in bpf_map__set_ifindex()
10247 	map->map_ifindex = ifindex;  in bpf_map__set_ifindex()
10253 	if (!bpf_map_type__is_map_in_map(map->def.type)) {  in bpf_map__set_inner_map_fd()
10255 		return libbpf_err(-EINVAL);  in bpf_map__set_inner_map_fd()
10257 	if (map->inner_map_fd != -1) {  in bpf_map__set_inner_map_fd()
10259 		return libbpf_err(-EINVAL);  in bpf_map__set_inner_map_fd()
10261 	if (map->inner_map) {  in bpf_map__set_inner_map_fd()
10262 		bpf_map__destroy(map->inner_map);  in bpf_map__set_inner_map_fd()
10263 		zfree(&map->inner_map);  in bpf_map__set_inner_map_fd()
10265 	map->inner_map_fd = fd;  in bpf_map__set_inner_map_fd()
10275 	if (!obj || !obj->maps)  in __bpf_map__iter()
10278 	s = obj->maps;  in __bpf_map__iter()
10279 	e = obj->maps + obj->nr_maps;  in __bpf_map__iter()
10287 	idx = (m - obj->maps) + i;  in __bpf_map__iter()
10288 	if (idx >= obj->nr_maps || idx < 0)  in __bpf_map__iter()
10290 	return &obj->maps[idx];  in __bpf_map__iter()
10297 		return obj->maps;  in bpf_object__next_map()
10306 		if (!obj->nr_maps)  in bpf_object__prev_map()
10308 		return obj->maps + obj->nr_maps - 1;  in bpf_object__prev_map()
10311 	return __bpf_map__iter(next, obj, -1);  in bpf_object__prev_map()
10325 			if (pos->real_name && strcmp(pos->real_name, name) == 0)  in bpf_object__find_map_by_name()
10331 			if (strcmp(pos->real_name, name) == 0)  in bpf_object__find_map_by_name()
10335 		if (strcmp(pos->name, name) == 0)  in bpf_object__find_map_by_name()
10351 		return -ENOENT;  in validate_map_op()
10353 	if (map->def.key_size != key_sz) {  in validate_map_op()
10355 			map->name, key_sz, map->def.key_size);  in validate_map_op()
10356 		return -EINVAL;  in validate_map_op()
10359 	if (map->fd < 0) {  in validate_map_op()
10360 		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);  in validate_map_op()
10361 		return -EINVAL;  in validate_map_op()
10367 	switch (map->def.type) {  in validate_map_op()
10373 		size_t elem_sz = roundup(map->def.value_size, 8);  in validate_map_op()
10376 …	pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n",  in validate_map_op()
10377 				map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz);  in validate_map_op()
10378 			return -EINVAL;  in validate_map_op()
10383 		if (map->def.value_size != value_sz) {  in validate_map_op()
10385 				map->name, value_sz, map->def.value_size);  in validate_map_op()
10386 			return -EINVAL;  in validate_map_op()
10403 	return bpf_map_lookup_elem_flags(map->fd, key, value, flags);  in bpf_map__lookup_elem()
10416 	return bpf_map_update_elem(map->fd, key, value, flags);  in bpf_map__update_elem()
10428 	return bpf_map_delete_elem_flags(map->fd, key, flags);  in bpf_map__delete_elem()
10441 	return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags);  in bpf_map__lookup_and_delete_elem()
10453 	return bpf_map_get_next_key(map->fd, cur_key, next_key);  in bpf_map__get_next_key()
10462 		errno = -PTR_ERR(ptr);  in libbpf_get_error()
10469 	return -errno;  in libbpf_get_error()
10480 			prog->name);  in bpf_link__update_program()
10481 		return libbpf_err(-EINVAL);  in bpf_link__update_program()
10500 	link->disconnected = true;  in bpf_link__disconnect()
10510 	if (!link->disconnected && link->detach)  in bpf_link__destroy()
10511 		err = link->detach(link);  in bpf_link__destroy()
10512 	if (link->pin_path)  in bpf_link__destroy()
10513 		free(link->pin_path);  in bpf_link__destroy()
10514 	if (link->dealloc)  in bpf_link__destroy()
10515 		link->dealloc(link);  in bpf_link__destroy()
10524 	return link->fd;  in bpf_link__fd()
10529 	return link->pin_path;  in bpf_link__pin_path()
10534 	return libbpf_err_errno(close(link->fd));  in bpf_link__detach_fd()
10544 		fd = -errno;  in bpf_link__open()
10552 		return libbpf_err_ptr(-ENOMEM);  in bpf_link__open()
10554 	link->detach = &bpf_link__detach_fd;  in bpf_link__open()
10555 	link->fd = fd;  in bpf_link__open()
10557 	link->pin_path = strdup(path);  in bpf_link__open()
10558 	if (!link->pin_path) {  in bpf_link__open()
10560 		return libbpf_err_ptr(-ENOMEM);  in bpf_link__open()
10568 	return bpf_link_detach(link->fd) ? -errno : 0;  in bpf_link__detach()
10575 	if (link->pin_path)  in bpf_link__pin()
10576 		return libbpf_err(-EBUSY);  in bpf_link__pin()
10584 	link->pin_path = strdup(path);  in bpf_link__pin()
10585 	if (!link->pin_path)  in bpf_link__pin()
10586 		return libbpf_err(-ENOMEM);  in bpf_link__pin()
10588 	if (bpf_obj_pin(link->fd, link->pin_path)) {  in bpf_link__pin()
10589 		err = -errno;  in bpf_link__pin()
10590 		zfree(&link->pin_path);  in bpf_link__pin()
10594 	pr_debug("link fd=%d: pinned at %s\n", link->fd, link->pin_path);  in bpf_link__pin()
10602 	if (!link->pin_path)  in bpf_link__unpin()
10603 		return libbpf_err(-EINVAL);  in bpf_link__unpin()
10605 	err = unlink(link->pin_path);  in bpf_link__unpin()
10607 		return -errno;  in bpf_link__unpin()
10609 	pr_debug("link fd=%d: unpinned from %s\n", link->fd, link->pin_path);  in bpf_link__unpin()
10610 	zfree(&link->pin_path);  in bpf_link__unpin()
10631 	if (ioctl(perf_link->perf_event_fd, PERF_EVENT_IOC_DISABLE, 0) < 0)  in bpf_link_perf_detach()
10632 		err = -errno;  in bpf_link_perf_detach()
10634 	if (perf_link->perf_event_fd != link->fd)  in bpf_link_perf_detach()
10635 		close(perf_link->perf_event_fd);  in bpf_link_perf_detach()
10636 	close(link->fd);  in bpf_link_perf_detach()
10639 	if (perf_link->legacy_probe_name) {  in bpf_link_perf_detach()
10640 		if (perf_link->legacy_is_kprobe) {  in bpf_link_perf_detach()
10641 			err = remove_kprobe_event_legacy(perf_link->legacy_probe_name,  in bpf_link_perf_detach()
10642 							 perf_link->legacy_is_retprobe);  in bpf_link_perf_detach()
10644 			err = remove_uprobe_event_legacy(perf_link->legacy_probe_name,  in bpf_link_perf_detach()
10645 							 perf_link->legacy_is_retprobe);  in bpf_link_perf_detach()
10656 	free(perf_link->legacy_probe_name);  in bpf_link_perf_dealloc()
10665 	int prog_fd, link_fd = -1, err;  in bpf_program__attach_perf_event_opts()
10669 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_perf_event_opts()
10673 			prog->name, pfd);  in bpf_program__attach_perf_event_opts()
10674 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_perf_event_opts()
10679 			prog->name);  in bpf_program__attach_perf_event_opts()
10680 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_perf_event_opts()
10685 		return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_perf_event_opts()
10686 	link->link.detach = &bpf_link_perf_detach;  in bpf_program__attach_perf_event_opts()
10687 	link->link.dealloc = &bpf_link_perf_dealloc;  in bpf_program__attach_perf_event_opts()
10688 	link->perf_event_fd = pfd;  in bpf_program__attach_perf_event_opts()
10691 	if (kernel_supports(prog->obj, FEAT_PERF_LINK) && !force_ioctl_attach) {  in bpf_program__attach_perf_event_opts()
10697 			err = -errno;  in bpf_program__attach_perf_event_opts()
10699 				prog->name, pfd,  in bpf_program__attach_perf_event_opts()
10703 		link->link.fd = link_fd;  in bpf_program__attach_perf_event_opts()
10706 			pr_warn("prog '%s': user context value is not supported\n", prog->name);  in bpf_program__attach_perf_event_opts()
10707 			err = -EOPNOTSUPP;  in bpf_program__attach_perf_event_opts()
10712 			err = -errno;  in bpf_program__attach_perf_event_opts()
10714 				prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));  in bpf_program__attach_perf_event_opts()
10715 			if (err == -EPROTO)  in bpf_program__attach_perf_event_opts()
10717 					prog->name, pfd);  in bpf_program__attach_perf_event_opts()
10720 		link->link.fd = pfd;  in bpf_program__attach_perf_event_opts()
10723 		err = -errno;  in bpf_program__attach_perf_event_opts()
10725 			prog->name, pfd, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));  in bpf_program__attach_perf_event_opts()
10729 	return &link->link;  in bpf_program__attach_perf_event_opts()
10743  * this function is expected to parse integer in the range of [0, 2^31-1] from
10755 		err = -errno;  in parse_uint_from_file()
10762 		err = err == EOF ? -EIO : -errno;  in parse_uint_from_file()
10812 		return -EINVAL;  in perf_event_open_probe()
10844 		      pid < 0 ? -1 : pid /* pid */,  in perf_event_open_probe()
10845 		      pid == -1 ? 0 : -1 /* cpu */,  in perf_event_open_probe()
10846 		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);  in perf_event_open_probe()
10847 	return pfd >= 0 ? pfd : -errno;  in perf_event_open_probe()
10861 		return -EINVAL;  in append_to_file()
10865 		return -errno;  in append_to_file()
10868 		err = -errno;  in append_to_file()
10879 	static int has_debugfs = -1;  in use_debugfs()
10941 	return append_to_file(tracefs_kprobe_events(), "-:%s/%s",  in remove_kprobe_event_legacy()
10985 		      pid < 0 ? -1 : pid, /* pid */  in perf_event_kprobe_open_legacy()
10986 		      pid == -1 ? 0 : -1, /* cpu */  in perf_event_kprobe_open_legacy()
10987 		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);  in perf_event_kprobe_open_legacy()
10989 		err = -errno;  in perf_event_kprobe_open_legacy()
11075 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_opts()
11090 			return libbpf_err_ptr(-ENOTSUP);  in bpf_program__attach_kprobe_opts()
11094 		if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))  in bpf_program__attach_kprobe_opts()
11095 			return libbpf_err_ptr(-ENOTSUP);  in bpf_program__attach_kprobe_opts()
11100 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_opts()
11106 					    -1 /* pid */, 0 /* ref_ctr_off */);  in bpf_program__attach_kprobe_opts()
11115 			return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_kprobe_opts()
11118 						    offset, -1 /* pid */);  in bpf_program__attach_kprobe_opts()
11121 		err = -errno;  in bpf_program__attach_kprobe_opts()
11123 			prog->name, retprobe ? "kretprobe" : "kprobe",  in bpf_program__attach_kprobe_opts()
11133 			prog->name, retprobe ? "kretprobe" : "kprobe",  in bpf_program__attach_kprobe_opts()
11141 		perf_link->legacy_probe_name = legacy_probe;  in bpf_program__attach_kprobe_opts()
11142 		perf_link->legacy_is_kprobe = true;  in bpf_program__attach_kprobe_opts()
11143 		perf_link->legacy_is_retprobe = retprobe;  in bpf_program__attach_kprobe_opts()
11175 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_ksyscall()
11177 	if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) {  in bpf_program__attach_ksyscall()
11244 	struct kprobe_multi_resolve *res = data->res;  in avail_kallsyms_cb()
11247 	if (!bsearch(&sym_name, data->syms, data->cnt, sizeof(*data->syms), avail_func_cmp))  in avail_kallsyms_cb()
11250 	err = libbpf_ensure_mem((void **)&res->addrs, &res->cap, sizeof(*res->addrs), res->cnt + 1);  in avail_kallsyms_cb()
11254 	res->addrs[res->cnt++] = (unsigned long)sym_addr;  in avail_kallsyms_cb()
11270 		err = -errno;  in libbpf_available_kallsyms_parse()
11284 			err = -EINVAL;  in libbpf_available_kallsyms_parse()
11288 		if (!glob_match(sym_name, res->pattern))  in libbpf_available_kallsyms_parse()
11297 			err = -errno;  in libbpf_available_kallsyms_parse()
11306 		err = -ENOENT;  in libbpf_available_kallsyms_parse()
11318 	if (res->cnt == 0)  in libbpf_available_kallsyms_parse()
11319 		err = -ENOENT;  in libbpf_available_kallsyms_parse()
11332 	return access(tracefs_available_filter_functions_addrs(), R_OK) != -1;  in has_available_filter_functions_addrs()
11345 		err = -errno;  in libbpf_available_kprobes_parse()
11358 			err = -EINVAL;  in libbpf_available_kprobes_parse()
11362 		if (!glob_match(sym_name, res->pattern))  in libbpf_available_kprobes_parse()
11365 		err = libbpf_ensure_mem((void **)&res->addrs, &res->cap,  in libbpf_available_kprobes_parse()
11366 					sizeof(*res->addrs), res->cnt + 1);  in libbpf_available_kprobes_parse()
11370 		res->addrs[res->cnt++] = (unsigned long)sym_addr;  in libbpf_available_kprobes_parse()
11373 	if (res->cnt == 0)  in libbpf_available_kprobes_parse()
11374 		err = -ENOENT;  in libbpf_available_kprobes_parse()
11400 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11405 			prog->name);  in bpf_program__attach_kprobe_multi_opts()
11406 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11415 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11417 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11419 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11421 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_kprobe_multi_opts()
11444 		err = -ENOMEM;  in bpf_program__attach_kprobe_multi_opts()
11447 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_kprobe_multi_opts()
11451 		err = -errno;  in bpf_program__attach_kprobe_multi_opts()
11453 			prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));  in bpf_program__attach_kprobe_multi_opts()
11456 	link->fd = link_fd;  in bpf_program__attach_kprobe_multi_opts()
11476 	/* no auto-attach for SEC("kprobe") and SEC("kretprobe") */  in attach_kprobe()
11477 	if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0)  in attach_kprobe()
11480 	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/");  in attach_kprobe()
11482 		func_name = prog->sec_name + sizeof("kretprobe/") - 1;  in attach_kprobe()
11484 		func_name = prog->sec_name + sizeof("kprobe/") - 1;  in attach_kprobe()
11486 	n = sscanf(func_name, "%m[a-zA-Z0-9_.]+%li", &func, &offset);  in attach_kprobe()
11489 		return -EINVAL;  in attach_kprobe()
11494 		return -EINVAL;  in attach_kprobe()
11510 	/* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */  in attach_ksyscall()
11511 	if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0)  in attach_ksyscall()
11514 	opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/");  in attach_ksyscall()
11516 		syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1;  in attach_ksyscall()
11518 		syscall_name = prog->sec_name + sizeof("ksyscall/") - 1;  in attach_ksyscall()
11521 	return *link ? 0 : -errno;  in attach_ksyscall()
11533 	/* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */  in attach_kprobe_multi()
11534 	if (strcmp(prog->sec_name, "kprobe.multi") == 0 ||  in attach_kprobe_multi()
11535 	    strcmp(prog->sec_name, "kretprobe.multi") == 0)  in attach_kprobe_multi()
11538 	opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/");  in attach_kprobe_multi()
11540 		spec = prog->sec_name + sizeof("kretprobe.multi/") - 1;  in attach_kprobe_multi()
11542 		spec = prog->sec_name + sizeof("kprobe.multi/") - 1;  in attach_kprobe_multi()
11544 	n = sscanf(spec, "%m[a-zA-Z0-9_.*?]", &pattern);  in attach_kprobe_multi()
11547 		return -EINVAL;  in attach_kprobe_multi()
11559 	int n, ret = -EINVAL;  in attach_uprobe_multi()
11563 	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",  in attach_uprobe_multi()
11567 		/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */  in attach_uprobe_multi()
11572 		*link = bpf_program__attach_uprobe_multi(prog, -1, binary_path, func_name, &opts);  in attach_uprobe_multi()
11576 		pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,  in attach_uprobe_multi()
11577 			prog->sec_name);  in attach_uprobe_multi()
11611 	return append_to_file(tracefs_uprobe_events(), "-:%s/%s",  in remove_uprobe_event_legacy()
11652 		      pid < 0 ? -1 : pid, /* pid */  in perf_event_uprobe_open_legacy()
11653 		      pid == -1 ? 0 : -1, /* cpu */  in perf_event_uprobe_open_legacy()
11654 		      -1 /* group_fd */,  PERF_FLAG_FD_CLOEXEC);  in perf_event_uprobe_open_legacy()
11656 		err = -errno;  in perf_event_uprobe_open_legacy()
11704 		ret = -LIBBPF_ERRNO__FORMAT;  in elf_find_func_offset_from_archive()
11711 			elf_errmsg(-1));  in elf_find_func_offset_from_archive()
11712 		ret = -LIBBPF_ERRNO__LIBELF;  in elf_find_func_offset_from_archive()
11739 	return "/lib/x86_64-linux-gnu";  in arch_specific_lib_paths()
11741 	return "/lib/i386-linux-gnu";  in arch_specific_lib_paths()
11743 	return "/lib/s390x-linux-gnu";  in arch_specific_lib_paths()
11745 	return "/lib/s390-linux-gnu";  in arch_specific_lib_paths()
11747 	return "/lib/arm-linux-gnueabi";  in arch_specific_lib_paths()
11749 	return "/lib/arm-linux-gnueabihf";  in arch_specific_lib_paths()
11751 	return "/lib/aarch64-linux-gnu";  in arch_specific_lib_paths()
11753 	return "/lib/mips64el-linux-gnuabi64";  in arch_specific_lib_paths()
11755 	return "/lib/mipsel-linux-gnu";  in arch_specific_lib_paths()
11757 	return "/lib/powerpc64le-linux-gnu";  in arch_specific_lib_paths()
11759 	return "/lib/sparc64-linux-gnu";  in arch_specific_lib_paths()
11761 	return "/lib/riscv64-linux-gnu";  in arch_specific_lib_paths()
11796 			seg_len = next_path ? next_path - s : strlen(s);  in resolve_full_path()
11807 	return -ENOENT;  in resolve_full_path()
11829 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11834 			prog->name);  in bpf_program__attach_uprobe_multi()
11835 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11852 	 *    - syms and offsets are mutually exclusive  in bpf_program__attach_uprobe_multi()
11853 	 *    - ref_ctr_offsets and cookies are optional  in bpf_program__attach_uprobe_multi()
11859 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11861 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11865 			return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11868 			return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_multi()
11876 					prog->name, path, err);  in bpf_program__attach_uprobe_multi()
11908 		err = -ENOMEM;  in bpf_program__attach_uprobe_multi()
11911 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_uprobe_multi()
11915 		err = -errno;  in bpf_program__attach_uprobe_multi()
11916 		pr_warn("prog '%s': failed to attach multi-uprobe: %s\n",  in bpf_program__attach_uprobe_multi()
11917 			prog->name, libbpf_strerror_r(err, errmsg, sizeof(errmsg)));  in bpf_program__attach_uprobe_multi()
11920 	link->fd = link_fd;  in bpf_program__attach_uprobe_multi()
11947 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_opts()
11955 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_opts()
11962 			       min(sizeof(full_path), (size_t)(archive_sep - binary_path + 1)));  in bpf_program__attach_uprobe_opts()
11969 				prog->name, binary_path, err);  in bpf_program__attach_uprobe_opts()
11998 			return libbpf_err_ptr(-ENOTSUP);  in bpf_program__attach_uprobe_opts()
12002 		if (legacy || !kernel_supports(prog->obj, FEAT_PERF_LINK))  in bpf_program__attach_uprobe_opts()
12003 			return libbpf_err_ptr(-ENOTSUP);  in bpf_program__attach_uprobe_opts()
12008 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_opts()
12018 			return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_uprobe_opts()
12025 			return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_uprobe_opts()
12031 		err = -errno;  in bpf_program__attach_uprobe_opts()
12033 			prog->name, retprobe ? "uretprobe" : "uprobe",  in bpf_program__attach_uprobe_opts()
12044 			prog->name, retprobe ? "uretprobe" : "uprobe",  in bpf_program__attach_uprobe_opts()
12052 		perf_link->legacy_probe_name = legacy_probe;  in bpf_program__attach_uprobe_opts()
12053 		perf_link->legacy_is_kprobe = false;  in bpf_program__attach_uprobe_opts()
12054 		perf_link->legacy_is_retprobe = retprobe;  in bpf_program__attach_uprobe_opts()
12066 /* Format of u[ret]probe section definition supporting auto-attach:
12073  * specified (and auto-attach is not possible) or the above format is specified for
12074  * auto-attach.
12080 	int n, c, ret = -EINVAL;  in attach_uprobe()
12085 	n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[^\n]",  in attach_uprobe()
12089 		/* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */  in attach_uprobe()
12094 			prog->name, prog->sec_name);  in attach_uprobe()
12112 				prog->name);  in attach_uprobe()
12116 		*link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts);  in attach_uprobe()
12120 		pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name,  in attach_uprobe()
12121 			prog->sec_name);  in attach_uprobe()
12147 	struct bpf_object *obj = prog->obj;  in bpf_program__attach_usdt()
12153 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_usdt()
12157 			prog->name);  in bpf_program__attach_usdt()
12158 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_usdt()
12162 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_usdt()
12168 				prog->name, binary_path, err);  in bpf_program__attach_usdt()
12177 	if (IS_ERR(obj->usdt_man))  in bpf_program__attach_usdt()
12178 		return libbpf_ptr(obj->usdt_man);  in bpf_program__attach_usdt()
12179 	if (!obj->usdt_man) {  in bpf_program__attach_usdt()
12180 		obj->usdt_man = usdt_manager_new(obj);  in bpf_program__attach_usdt()
12181 		if (IS_ERR(obj->usdt_man))  in bpf_program__attach_usdt()
12182 			return libbpf_ptr(obj->usdt_man);  in bpf_program__attach_usdt()
12186 	link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path,  in bpf_program__attach_usdt()
12202 		/* no auto-attach for just SEC("usdt") */  in attach_usdt()
12211 		err = -EINVAL;  in attach_usdt()
12213 		*link = bpf_program__attach_usdt(prog, -1 /* any process */, path,  in attach_usdt()
12232 		return -errno;  in determine_tracepoint_id()
12236 		return -E2BIG;  in determine_tracepoint_id()
12262 	pfd = syscall(__NR_perf_event_open, &attr, -1 /* pid */, 0 /* cpu */,  in perf_event_open_tracepoint()
12263 		      -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC);  in perf_event_open_tracepoint()
12265 		err = -errno;  in perf_event_open_tracepoint()
12285 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_tracepoint_opts()
12292 			prog->name, tp_category, tp_name,  in bpf_program__attach_tracepoint_opts()
12301 			prog->name, tp_category, tp_name,  in bpf_program__attach_tracepoint_opts()
12321 	/* no auto-attach for SEC("tp") or SEC("tracepoint") */  in attach_tp()
12322 	if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0)  in attach_tp()
12325 	sec_name = strdup(prog->sec_name);  in attach_tp()
12327 		return -ENOMEM;  in attach_tp()
12330 	if (str_has_pfx(prog->sec_name, "tp/"))  in attach_tp()
12331 		tp_cat = sec_name + sizeof("tp/") - 1;  in attach_tp()
12333 		tp_cat = sec_name + sizeof("tracepoint/") - 1;  in attach_tp()
12337 		return -EINVAL;  in attach_tp()
12358 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_raw_tracepoint_opts()
12362 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);  in bpf_program__attach_raw_tracepoint_opts()
12363 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_raw_tracepoint_opts()
12368 		return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_raw_tracepoint_opts()
12369 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_raw_tracepoint_opts()
12375 		pfd = -errno;  in bpf_program__attach_raw_tracepoint_opts()
12378 			prog->name, tp_name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));  in bpf_program__attach_raw_tracepoint_opts()
12381 	link->fd = pfd;  in bpf_program__attach_raw_tracepoint_opts()
12407 		if (!str_has_pfx(prog->sec_name, prefixes[i]))  in attach_raw_tp()
12411 		/* no auto-attach case of, e.g., SEC("raw_tp") */  in attach_raw_tp()
12412 		if (prog->sec_name[pfx_len] == '\0')  in attach_raw_tp()
12415 		if (prog->sec_name[pfx_len] != '/')  in attach_raw_tp()
12418 		tp_name = prog->sec_name + pfx_len + 1;  in attach_raw_tp()
12424 			prog->name, prog->sec_name);  in attach_raw_tp()
12425 		return -EINVAL;  in attach_raw_tp()
12442 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_btf_id()
12446 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);  in bpf_program__attach_btf_id()
12447 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_btf_id()
12452 		return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_btf_id()
12453 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_btf_id()
12459 		pfd = -errno;  in bpf_program__attach_btf_id()
12462 			prog->name, libbpf_strerror_r(pfd, errmsg, sizeof(errmsg)));  in bpf_program__attach_btf_id()
12465 	link->fd = pfd;  in bpf_program__attach_btf_id()
12509 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);  in bpf_program_attach_fd()
12510 		return libbpf_err_ptr(-EINVAL);  in bpf_program_attach_fd()
12515 		return libbpf_err_ptr(-ENOMEM);  in bpf_program_attach_fd()
12516 	link->detach = &bpf_link__detach_fd;  in bpf_program_attach_fd()
12521 		link_fd = -errno;  in bpf_program_attach_fd()
12524 			prog->name, target_name,  in bpf_program_attach_fd()
12528 	link->fd = link_fd;  in bpf_program_attach_fd()
12559 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_tcx()
12564 	/* validate we don't have unexpected combinations of non-zero fields */  in bpf_program__attach_tcx()
12567 			prog->name);  in bpf_program__attach_tcx()
12568 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_tcx()
12572 			prog->name);  in bpf_program__attach_tcx()
12573 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_tcx()
12594 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_netkit()
12599 	/* validate we don't have unexpected combinations of non-zero fields */  in bpf_program__attach_netkit()
12602 			prog->name);  in bpf_program__attach_netkit()
12603 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_netkit()
12607 			prog->name);  in bpf_program__attach_netkit()
12608 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_netkit()
12627 			prog->name);  in bpf_program__attach_freplace()
12628 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_freplace()
12631 	if (prog->type != BPF_PROG_TYPE_EXT) {  in bpf_program__attach_freplace()
12633 			prog->name);  in bpf_program__attach_freplace()
12634 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_freplace()
12667 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_iter()
12674 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);  in bpf_program__attach_iter()
12675 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_iter()
12680 		return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_iter()
12681 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_iter()
12686 		link_fd = -errno;  in bpf_program__attach_iter()
12689 			prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));  in bpf_program__attach_iter()
12692 	link->fd = link_fd;  in bpf_program__attach_iter()
12710 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_netfilter()
12714 		pr_warn("prog '%s': can't attach before loaded\n", prog->name);  in bpf_program__attach_netfilter()
12715 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach_netfilter()
12720 		return libbpf_err_ptr(-ENOMEM);  in bpf_program__attach_netfilter()
12722 	link->detach = &bpf_link__detach_fd;  in bpf_program__attach_netfilter()
12733 		link_fd = -errno;  in bpf_program__attach_netfilter()
12736 			prog->name, libbpf_strerror_r(link_fd, errmsg, sizeof(errmsg)));  in bpf_program__attach_netfilter()
12739 	link->fd = link_fd;  in bpf_program__attach_netfilter()
12749 	if (!prog->sec_def || !prog->sec_def->prog_attach_fn)  in bpf_program__attach()
12750 		return libbpf_err_ptr(-EOPNOTSUPP);  in bpf_program__attach()
12754 			prog->name);  in bpf_program__attach()
12755 		return libbpf_err_ptr(-EINVAL);  in bpf_program__attach()
12758 	err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, &link);  in bpf_program__attach()
12762 	/* When calling bpf_program__attach() explicitly, auto-attach support  in bpf_program__attach()
12768 		return libbpf_err_ptr(-EOPNOTSUPP);  in bpf_program__attach()
12785 	if (st_link->map_fd < 0)  in bpf_link__detach_struct_ops()
12787 		return bpf_map_delete_elem(link->fd, &zero);  in bpf_link__detach_struct_ops()
12789 	return close(link->fd);  in bpf_link__detach_struct_ops()
12799 		return libbpf_err_ptr(-EINVAL);  in bpf_map__attach_struct_ops()
12801 	if (map->fd < 0) {  in bpf_map__attach_struct_ops()
12802 		pr_warn("map '%s': can't attach BPF map without FD (was it created?)\n", map->name);  in bpf_map__attach_struct_ops()
12803 		return libbpf_err_ptr(-EINVAL);  in bpf_map__attach_struct_ops()
12808 		return libbpf_err_ptr(-EINVAL);  in bpf_map__attach_struct_ops()
12811 	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);  in bpf_map__attach_struct_ops()
12817 	if (err && (!(map->def.map_flags & BPF_F_LINK) || err != -EBUSY)) {  in bpf_map__attach_struct_ops()
12822 	link->link.detach = bpf_link__detach_struct_ops;  in bpf_map__attach_struct_ops()
12824 	if (!(map->def.map_flags & BPF_F_LINK)) {  in bpf_map__attach_struct_ops()
12826 		link->link.fd = map->fd;  in bpf_map__attach_struct_ops()
12827 		link->map_fd = -1;  in bpf_map__attach_struct_ops()
12828 		return &link->link;  in bpf_map__attach_struct_ops()
12831 	fd = bpf_link_create(map->fd, 0, BPF_STRUCT_OPS, NULL);  in bpf_map__attach_struct_ops()
12837 	link->link.fd = fd;  in bpf_map__attach_struct_ops()
12838 	link->map_fd = map->fd;  in bpf_map__attach_struct_ops()
12840 	return &link->link;  in bpf_map__attach_struct_ops()
12853 		return -EINVAL;  in bpf_link__update_map()
12855 	if (map->fd < 0) {  in bpf_link__update_map()
12856 		pr_warn("map '%s': can't use BPF map without FD (was it created?)\n", map->name);  in bpf_link__update_map()
12857 		return -EINVAL;  in bpf_link__update_map()
12862 	if (st_ops_link->map_fd < 0)  in bpf_link__update_map()
12863 		return -EINVAL;  in bpf_link__update_map()
12865 	err = bpf_map_update_elem(map->fd, &zero, map->st_ops->kern_vdata, 0);  in bpf_link__update_map()
12871 	if (err && err != -EBUSY)  in bpf_link__update_map()
12874 	err = bpf_link_update(link->fd, map->fd, NULL);  in bpf_link__update_map()
12878 	st_ops_link->map_fd = map->fd;  in bpf_link__update_map()
12893 	__u64 data_tail = header->data_tail;  in perf_event_read_simple()
12900 		ehdr = base + (data_tail & (mmap_size - 1));  in perf_event_read_simple()
12901 		ehdr_size = ehdr->size;  in perf_event_read_simple()
12905 			size_t len_first = base + mmap_size - copy_start;  in perf_event_read_simple()
12906 			size_t len_secnd = ehdr_size - len_first;  in perf_event_read_simple()
12940 	/* sample_cb and lost_cb are higher-level common-case callbacks */
12979 	if (cpu_buf->base &&  in perf_buffer__free_cpu_buf()
12980 	    munmap(cpu_buf->base, pb->mmap_size + pb->page_size))  in perf_buffer__free_cpu_buf()
12981 		pr_warn("failed to munmap cpu_buf #%d\n", cpu_buf->cpu);  in perf_buffer__free_cpu_buf()
12982 	if (cpu_buf->fd >= 0) {  in perf_buffer__free_cpu_buf()
12983 		ioctl(cpu_buf->fd, PERF_EVENT_IOC_DISABLE, 0);  in perf_buffer__free_cpu_buf()
12984 		close(cpu_buf->fd);  in perf_buffer__free_cpu_buf()
12986 	free(cpu_buf->buf);  in perf_buffer__free_cpu_buf()
12996 	if (pb->cpu_bufs) {  in perf_buffer__free()
12997 		for (i = 0; i < pb->cpu_cnt; i++) {  in perf_buffer__free()
12998 			struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__free()
13003 			bpf_map_delete_elem(pb->map_fd, &cpu_buf->map_key);  in perf_buffer__free()
13006 		free(pb->cpu_bufs);  in perf_buffer__free()
13008 	if (pb->epoll_fd >= 0)  in perf_buffer__free()
13009 		close(pb->epoll_fd);  in perf_buffer__free()
13010 	free(pb->events);  in perf_buffer__free()
13024 		return ERR_PTR(-ENOMEM);  in perf_buffer__open_cpu_buf()
13026 	cpu_buf->pb = pb;  in perf_buffer__open_cpu_buf()
13027 	cpu_buf->cpu = cpu;  in perf_buffer__open_cpu_buf()
13028 	cpu_buf->map_key = map_key;  in perf_buffer__open_cpu_buf()
13030 	cpu_buf->fd = syscall(__NR_perf_event_open, attr, -1 /* pid */, cpu,  in perf_buffer__open_cpu_buf()
13031 			      -1, PERF_FLAG_FD_CLOEXEC);  in perf_buffer__open_cpu_buf()
13032 	if (cpu_buf->fd < 0) {  in perf_buffer__open_cpu_buf()
13033 		err = -errno;  in perf_buffer__open_cpu_buf()
13039 	cpu_buf->base = mmap(NULL, pb->mmap_size + pb->page_size,  in perf_buffer__open_cpu_buf()
13041 			     cpu_buf->fd, 0);  in perf_buffer__open_cpu_buf()
13042 	if (cpu_buf->base == MAP_FAILED) {  in perf_buffer__open_cpu_buf()
13043 		cpu_buf->base = NULL;  in perf_buffer__open_cpu_buf()
13044 		err = -errno;  in perf_buffer__open_cpu_buf()
13050 	if (ioctl(cpu_buf->fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {  in perf_buffer__open_cpu_buf()
13051 		err = -errno;  in perf_buffer__open_cpu_buf()
13079 		return libbpf_err_ptr(-EINVAL);  in perf_buffer__new()
13109 		return libbpf_err_ptr(-EINVAL);  in perf_buffer__new_raw()
13112 		return libbpf_err_ptr(-EINVAL);  in perf_buffer__new_raw()
13135 	if (page_cnt == 0 || (page_cnt & (page_cnt - 1))) {  in __perf_buffer__new()
13138 		return ERR_PTR(-EINVAL);  in __perf_buffer__new()
13141 	/* best-effort sanity checks */  in __perf_buffer__new()
13146 		err = -errno;  in __perf_buffer__new()
13148 		 * -EBADFD, -EFAULT, or -E2BIG on real error  in __perf_buffer__new()
13150 		if (err != -EINVAL) {  in __perf_buffer__new()
13161 			return ERR_PTR(-EINVAL);  in __perf_buffer__new()
13167 		return ERR_PTR(-ENOMEM);  in __perf_buffer__new()
13169 	pb->event_cb = p->event_cb;  in __perf_buffer__new()
13170 	pb->sample_cb = p->sample_cb;  in __perf_buffer__new()
13171 	pb->lost_cb = p->lost_cb;  in __perf_buffer__new()
13172 	pb->ctx = p->ctx;  in __perf_buffer__new()
13174 	pb->page_size = getpagesize();  in __perf_buffer__new()
13175 	pb->mmap_size = pb->page_size * page_cnt;  in __perf_buffer__new()
13176 	pb->map_fd = map_fd;  in __perf_buffer__new()
13178 	pb->epoll_fd = epoll_create1(EPOLL_CLOEXEC);  in __perf_buffer__new()
13179 	if (pb->epoll_fd < 0) {  in __perf_buffer__new()
13180 		err = -errno;  in __perf_buffer__new()
13186 	if (p->cpu_cnt > 0) {  in __perf_buffer__new()
13187 		pb->cpu_cnt = p->cpu_cnt;  in __perf_buffer__new()
13189 		pb->cpu_cnt = libbpf_num_possible_cpus();  in __perf_buffer__new()
13190 		if (pb->cpu_cnt < 0) {  in __perf_buffer__new()
13191 			err = pb->cpu_cnt;  in __perf_buffer__new()
13194 		if (map.max_entries && map.max_entries < pb->cpu_cnt)  in __perf_buffer__new()
13195 			pb->cpu_cnt = map.max_entries;  in __perf_buffer__new()
13198 	pb->events = calloc(pb->cpu_cnt, sizeof(*pb->events));  in __perf_buffer__new()
13199 	if (!pb->events) {  in __perf_buffer__new()
13200 		err = -ENOMEM;  in __perf_buffer__new()
13204 	pb->cpu_bufs = calloc(pb->cpu_cnt, sizeof(*pb->cpu_bufs));  in __perf_buffer__new()
13205 	if (!pb->cpu_bufs) {  in __perf_buffer__new()
13206 		err = -ENOMEM;  in __perf_buffer__new()
13217 	for (i = 0, j = 0; i < pb->cpu_cnt; i++) {  in __perf_buffer__new()
13221 		cpu = p->cpu_cnt > 0 ? p->cpus[i] : i;  in __perf_buffer__new()
13222 		map_key = p->cpu_cnt > 0 ? p->map_keys[i] : i;  in __perf_buffer__new()
13227 		if (p->cpu_cnt <= 0 && (cpu >= n || !online[cpu]))  in __perf_buffer__new()
13230 		cpu_buf = perf_buffer__open_cpu_buf(pb, p->attr, cpu, map_key);  in __perf_buffer__new()
13236 		pb->cpu_bufs[j] = cpu_buf;  in __perf_buffer__new()
13238 		err = bpf_map_update_elem(pb->map_fd, &map_key,  in __perf_buffer__new()
13239 					  &cpu_buf->fd, 0);  in __perf_buffer__new()
13241 			err = -errno;  in __perf_buffer__new()
13242 			pr_warn("failed to set cpu #%d, key %d -> perf FD %d: %s\n",  in __perf_buffer__new()
13243 				cpu, map_key, cpu_buf->fd,  in __perf_buffer__new()
13248 		pb->events[j].events = EPOLLIN;  in __perf_buffer__new()
13249 		pb->events[j].data.ptr = cpu_buf;  in __perf_buffer__new()
13250 		if (epoll_ctl(pb->epoll_fd, EPOLL_CTL_ADD, cpu_buf->fd,  in __perf_buffer__new()
13251 			      &pb->events[j]) < 0) {  in __perf_buffer__new()
13252 			err = -errno;  in __perf_buffer__new()
13254 				cpu, cpu_buf->fd,  in __perf_buffer__new()
13260 	pb->cpu_cnt = j;  in __perf_buffer__new()
13289 	struct perf_buffer *pb = cpu_buf->pb;  in perf_buffer__process_record()
13293 	if (pb->event_cb)  in perf_buffer__process_record()
13294 		return pb->event_cb(pb->ctx, cpu_buf->cpu, e);  in perf_buffer__process_record()
13296 	switch (e->type) {  in perf_buffer__process_record()
13300 		if (pb->sample_cb)  in perf_buffer__process_record()
13301 			pb->sample_cb(pb->ctx, cpu_buf->cpu, s->data, s->size);  in perf_buffer__process_record()
13307 		if (pb->lost_cb)  in perf_buffer__process_record()
13308 			pb->lost_cb(pb->ctx, cpu_buf->cpu, s->lost);  in perf_buffer__process_record()
13312 		pr_warn("unknown perf sample type %d\n", e->type);  in perf_buffer__process_record()
13323 	ret = perf_event_read_simple(cpu_buf->base, pb->mmap_size,  in perf_buffer__process_records()
13324 				     pb->page_size, &cpu_buf->buf,  in perf_buffer__process_records()
13325 				     &cpu_buf->buf_size,  in perf_buffer__process_records()
13334 	return pb->epoll_fd;  in perf_buffer__epoll_fd()
13341 	cnt = epoll_wait(pb->epoll_fd, pb->events, pb->cpu_cnt, timeout_ms);  in perf_buffer__poll()
13343 		return -errno;  in perf_buffer__poll()
13346 		struct perf_cpu_buf *cpu_buf = pb->events[i].data.ptr;  in perf_buffer__poll()
13362 	return pb->cpu_cnt;  in perf_buffer__buffer_cnt()
13374 	if (buf_idx >= pb->cpu_cnt)  in perf_buffer__buffer_fd()
13375 		return libbpf_err(-EINVAL);  in perf_buffer__buffer_fd()
13377 	cpu_buf = pb->cpu_bufs[buf_idx];  in perf_buffer__buffer_fd()
13379 		return libbpf_err(-ENOENT);  in perf_buffer__buffer_fd()
13381 	return cpu_buf->fd;  in perf_buffer__buffer_fd()
13388 	if (buf_idx >= pb->cpu_cnt)  in perf_buffer__buffer()
13389 		return libbpf_err(-EINVAL);  in perf_buffer__buffer()
13391 	cpu_buf = pb->cpu_bufs[buf_idx];  in perf_buffer__buffer()
13393 		return libbpf_err(-ENOENT);  in perf_buffer__buffer()
13395 	*buf = cpu_buf->base;  in perf_buffer__buffer()
13396 	*buf_size = pb->mmap_size;  in perf_buffer__buffer()
13405  *   - 0 on success;
13406  *   - <0 on failure.
13412 	if (buf_idx >= pb->cpu_cnt)  in perf_buffer__consume_buffer()
13413 		return libbpf_err(-EINVAL);  in perf_buffer__consume_buffer()
13415 	cpu_buf = pb->cpu_bufs[buf_idx];  in perf_buffer__consume_buffer()
13417 		return libbpf_err(-ENOENT);  in perf_buffer__consume_buffer()
13426 	for (i = 0; i < pb->cpu_cnt; i++) {  in perf_buffer__consume()
13427 		struct perf_cpu_buf *cpu_buf = pb->cpu_bufs[i];  in perf_buffer__consume()
13448 		return libbpf_err(-EINVAL);  in bpf_program__set_attach_target()
13450 	if (prog->obj->loaded)  in bpf_program__set_attach_target()
13451 		return libbpf_err(-EINVAL);  in bpf_program__set_attach_target()
13457 		prog->attach_prog_fd = attach_prog_fd;  in bpf_program__set_attach_target()
13468 			return libbpf_err(-EINVAL);  in bpf_program__set_attach_target()
13471 		err = bpf_object__load_vmlinux_btf(prog->obj, true);  in bpf_program__set_attach_target()
13474 		err = find_kernel_btf_id(prog->obj, attach_func_name,  in bpf_program__set_attach_target()
13475 					 prog->expected_attach_type,  in bpf_program__set_attach_target()
13481 	prog->attach_btf_id = btf_id;  in bpf_program__set_attach_target()
13482 	prog->attach_btf_obj_fd = btf_obj_fd;  in bpf_program__set_attach_target()
13483 	prog->attach_prog_fd = attach_prog_fd;  in bpf_program__set_attach_target()
13489 	int err = 0, n, len, start, end = -1;  in parse_cpu_mask_str()
13495 	/* Each sub string separated by ',' has format \d+-\d+ or \d+ */  in parse_cpu_mask_str()
13501 		n = sscanf(s, "%d%n-%d%n", &start, &len, &end, &len);  in parse_cpu_mask_str()
13504 			err = -EINVAL;  in parse_cpu_mask_str()
13512 			err = -EINVAL;  in parse_cpu_mask_str()
13517 			err = -ENOMEM;  in parse_cpu_mask_str()
13521 		memset(tmp + *mask_sz, 0, start - *mask_sz);  in parse_cpu_mask_str()
13522 		memset(tmp + start, 1, end - start + 1);  in parse_cpu_mask_str()
13528 		return -EINVAL;  in parse_cpu_mask_str()
13544 		err = -errno;  in parse_cpu_mask_file()
13551 		err = len ? -errno : -EINVAL;  in parse_cpu_mask_file()
13557 		return -E2BIG;  in parse_cpu_mask_file()
13598 		struct bpf_map **map = map_skel->map;  in populate_skeleton_maps()
13599 		const char *name = map_skel->name;  in populate_skeleton_maps()
13600 		void **mmaped = map_skel->mmaped;  in populate_skeleton_maps()
13605 			return -ESRCH;  in populate_skeleton_maps()
13608 		/* externs shouldn't be pre-setup from user code */  in populate_skeleton_maps()
13609 		if (mmaped && (*map)->libbpf_type != LIBBPF_MAP_KCONFIG)  in populate_skeleton_maps()
13610 			*mmaped = (*map)->mmaped;  in populate_skeleton_maps()
13623 		struct bpf_program **prog = prog_skel->prog;  in populate_skeleton_progs()
13624 		const char *name = prog_skel->name;  in populate_skeleton_progs()
13629 			return -ESRCH;  in populate_skeleton_progs()
13639 		.object_name = s->name,  in bpf_object__open_skeleton()
13644 	/* Attempt to preserve opts->object_name, unless overriden by user  in bpf_object__open_skeleton()
13652 		if (!opts->object_name)  in bpf_object__open_skeleton()
13653 			skel_opts.object_name = s->name;  in bpf_object__open_skeleton()
13656 	obj = bpf_object__open_mem(s->data, s->data_sz, &skel_opts);  in bpf_object__open_skeleton()
13660 			s->name, err);  in bpf_object__open_skeleton()
13664 	*s->obj = obj;  in bpf_object__open_skeleton()
13665 	err = populate_skeleton_maps(obj, s->maps, s->map_cnt, s->map_skel_sz);  in bpf_object__open_skeleton()
13667 		pr_warn("failed to populate skeleton maps for '%s': %d\n", s->name, err);  in bpf_object__open_skeleton()
13671 	err = populate_skeleton_progs(obj, s->progs, s->prog_cnt, s->prog_skel_sz);  in bpf_object__open_skeleton()
13673 		pr_warn("failed to populate skeleton progs for '%s': %d\n", s->name, err);  in bpf_object__open_skeleton()
13691 	if (!s->obj)  in bpf_object__open_subskeleton()
13692 		return libbpf_err(-EINVAL);  in bpf_object__open_subskeleton()
13694 	btf = bpf_object__btf(s->obj);  in bpf_object__open_subskeleton()
13697 			bpf_object__name(s->obj));  in bpf_object__open_subskeleton()
13698 		return libbpf_err(-errno);  in bpf_object__open_subskeleton()
13701 	err = populate_skeleton_maps(s->obj, s->maps, s->map_cnt, s->map_skel_sz);  in bpf_object__open_subskeleton()
13707 	err = populate_skeleton_progs(s->obj, s->progs, s->prog_cnt, s->prog_skel_sz);  in bpf_object__open_subskeleton()
13713 	for (var_idx = 0; var_idx < s->var_cnt; var_idx++) {  in bpf_object__open_subskeleton()
13714 		var_skel = (void *)s->vars + var_idx * s->var_skel_sz;  in bpf_object__open_subskeleton()
13715 		map = *var_skel->map;  in bpf_object__open_subskeleton()
13723 			return libbpf_err(-EINVAL);  in bpf_object__open_subskeleton()
13729 			var_type = btf__type_by_id(btf, var->type);  in bpf_object__open_subskeleton()
13730 			var_name = btf__name_by_offset(btf, var_type->name_off);  in bpf_object__open_subskeleton()
13731 			if (strcmp(var_name, var_skel->name) == 0) {  in bpf_object__open_subskeleton()
13732 				*var_skel->addr = map->mmaped + var->offset;  in bpf_object__open_subskeleton()
13744 	free(s->maps);  in bpf_object__destroy_subskeleton()
13745 	free(s->progs);  in bpf_object__destroy_subskeleton()
13746 	free(s->vars);  in bpf_object__destroy_subskeleton()
13754 	err = bpf_object__load(*s->obj);  in bpf_object__load_skeleton()
13756 		pr_warn("failed to load BPF skeleton '%s': %d\n", s->name, err);  in bpf_object__load_skeleton()
13760 	for (i = 0; i < s->map_cnt; i++) {  in bpf_object__load_skeleton()
13761 		struct bpf_map_skeleton *map_skel = (void *)s->maps + i * s->map_skel_sz;  in bpf_object__load_skeleton()
13762 		struct bpf_map *map = *map_skel->map;  in bpf_object__load_skeleton()
13764 		int prot, map_fd = map->fd;  in bpf_object__load_skeleton()
13765 		void **mmaped = map_skel->mmaped;  in bpf_object__load_skeleton()
13770 		if (!(map->def.map_flags & BPF_F_MMAPABLE)) {  in bpf_object__load_skeleton()
13775 		if (map->def.type == BPF_MAP_TYPE_ARENA) {  in bpf_object__load_skeleton()
13776 			*mmaped = map->mmaped;  in bpf_object__load_skeleton()
13780 		if (map->def.map_flags & BPF_F_RDONLY_PROG)  in bpf_object__load_skeleton()
13785 		/* Remap anonymous mmap()-ed "map initialization image" as  in bpf_object__load_skeleton()
13786 		 * a BPF map-backed mmap()-ed memory, but preserving the same  in bpf_object__load_skeleton()
13795 		*mmaped = mmap(map->mmaped, mmap_sz, prot, MAP_SHARED | MAP_FIXED, map_fd, 0);  in bpf_object__load_skeleton()
13797 			err = -errno;  in bpf_object__load_skeleton()
13799 			pr_warn("failed to re-mmap() map '%s': %d\n",  in bpf_object__load_skeleton()
13812 	for (i = 0; i < s->prog_cnt; i++) {  in bpf_object__attach_skeleton()
13813 		struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;  in bpf_object__attach_skeleton()
13814 		struct bpf_program *prog = *prog_skel->prog;  in bpf_object__attach_skeleton()
13815 		struct bpf_link **link = prog_skel->link;  in bpf_object__attach_skeleton()
13817 		if (!prog->autoload || !prog->autoattach)  in bpf_object__attach_skeleton()
13820 		/* auto-attaching not supported for this program */  in bpf_object__attach_skeleton()
13821 		if (!prog->sec_def || !prog->sec_def->prog_attach_fn)  in bpf_object__attach_skeleton()
13824 		/* if user already set the link manually, don't attempt auto-attach */  in bpf_object__attach_skeleton()
13828 		err = prog->sec_def->prog_attach_fn(prog, prog->sec_def->cookie, link);  in bpf_object__attach_skeleton()
13830 			pr_warn("prog '%s': failed to auto-attach: %d\n",  in bpf_object__attach_skeleton()
13835 		/* It's possible that for some SEC() definitions auto-attach  in bpf_object__attach_skeleton()
13840 		 * auto-attached. But if not, it shouldn't trigger skeleton's  in bpf_object__attach_skeleton()
13854 	for (i = 0; i < s->prog_cnt; i++) {  in bpf_object__detach_skeleton()
13855 		struct bpf_prog_skeleton *prog_skel = (void *)s->progs + i * s->prog_skel_sz;  in bpf_object__detach_skeleton()
13856 		struct bpf_link **link = prog_skel->link;  in bpf_object__detach_skeleton()
13868 	if (s->progs)  in bpf_object__destroy_skeleton()
13870 	if (s->obj)  in bpf_object__destroy_skeleton()
13871 		bpf_object__close(*s->obj);  in bpf_object__destroy_skeleton()
13872 	free(s->maps);  in bpf_object__destroy_skeleton()
13873 	free(s->progs);  in bpf_object__destroy_skeleton()