Lines Matching +full:ia32 +full:- +full:3 +full:a
1 // SPDX-License-Identifier: GPL-2.0
3 * Just-In-Time compiler for eBPF filters on IA32 (32bit x86)
17 #include <asm/nospec-branch.h>
18 #include <asm/asm-prototypes.h>
25 * original ESP => +-----+
27 * +-----+
29 * BPF_FP,IA32_EBP => +-----+
31 * +-----+
33 * current ESP => +-----+
37 * +-----+
43 * original ESP => +------------------+ \
45 * current EBP => +------------------+ } callee saved registers
47 * +------------------+ /
69 #define EMIT3(b1, b2, b3) EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
82 #define jmp_label(label, jmp_insn_len) (label - cnt - jmp_insn_len)
86 return value <= 127 && value >= -128; in is_imm8()
124 * Map eBPF registers to IA32 32bit registers or stack scratch space.
126 * 1. All the registers, R0-R10, are mapped to scratch space on stack.
129 * 3. For performance reason, the BPF_REG_AX for blinding constant, is
132 * As the eBPF registers are all 64 bit registers and IA32 has only 32 bit
133 * registers, we have to map each eBPF registers with two IA32 32 bit regs
139 /* Return value from in-kernel function, and exit value from eBPF */
142 /* The arguments from eBPF program to in-kernel function */
150 /* Callee saved registers that in-kernel function will preserve */
188 /* Encode 'dst_reg' register into IA32 opcode 'byte' */
194 /* Encode 'dst_reg' and 'src_reg' registers into IA32 opcode 'byte' */
197 return byte + dst_reg + (src_reg << 3); in add_2reg()
264 else if (!aux->verifier_zext) in emit_ia32_mov_r64()
335 * Emit 'movzwl eax,ax' to zero extend 16-bit in emit_ia32_to_le_r64()
340 if (!aux->verifier_zext) in emit_ia32_to_le_r64()
345 if (!aux->verifier_zext) in emit_ia32_to_le_r64()
389 if (!aux->verifier_zext) in emit_ia32_to_be_r64()
398 if (!aux->verifier_zext) in emit_ia32_to_be_r64()
548 /* dst = dst - src */ in emit_ia32_alu_r()
588 else if (!aux->verifier_zext) in emit_ia32_alu_r64()
629 /* dst = dst - val */ in emit_ia32_alu_i()
691 else if (!aux->verifier_zext) in emit_ia32_alu_i64()
911 u32 value = val - 32; in emit_ia32_lsh_i64()
960 u32 value = val - 32; in emit_ia32_rsh_i64()
1008 u32 value = val - 32; in emit_ia32_arsh_i64()
1259 /* mov ebx,dword ptr [ebp-12]*/ in emit_epilogue()
1260 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12); in emit_epilogue()
1261 /* mov esi,dword ptr [ebp-8]*/ in emit_epilogue()
1262 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8); in emit_epilogue()
1263 /* mov edi,dword ptr [ebp-4]*/ in emit_epilogue()
1264 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4); in emit_epilogue()
1277 EMIT1_off32(0xE9, (u8 *)__x86_indirect_thunk_edx - (ip + 5)); in emit_jmp_edx()
1289 * if (index >= array->map.max_entries)
1293 * prog = array->ptrs[index];
1296 * goto *(prog->bpf_func + prologue_size);
1308 static int jmp_label1 = -1; in emit_bpf_tail_call()
1311 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
1336 EMIT2(IA32_JNE, 3); in emit_bpf_tail_call()
1353 /* prog = array->ptrs[index]; */ in emit_bpf_tail_call()
1366 /* goto *(prog->bpf_func + prologue_size); */ in emit_bpf_tail_call()
1379 * edx == prog->bpf_func + prologue_size in emit_bpf_tail_call()
1383 if (jmp_label1 == -1) in emit_bpf_tail_call()
1491 /* i386 kernel compiles with "-mregparm=3". From gcc document:
1495 * On x86-32 targets, the regparm attribute causes the compiler
1498 * Functions that take a variable number of arguments continue
1502 * The first three args of a function will be considered for
1505 * Two 32bit registers are used to pass a 64bit arg.
1508 * void foo(u32 a, u32 b, u32 c, u32 d):
1509 * u32 a: EAX
1514 * void foo(u64 a, u32 b, u32 c):
1515 * u64 a: EAX (lo32) EDX (hi32)
1519 * void foo(u32 a, u64 b, u32 c):
1520 * u32 a: EAX
1524 * void foo(u32 a, u32 b, u64 c):
1525 * u32 a: EAX
1532 * u32 foo(u32 a, u32 b, u32 c):
1535 * u64 foo(u32 a, u32 b, u32 c):
1541 * struct-by-value.
1544 * bpf_jit_find_kfunc_model(). A btf_func_model
1557 * When emitting a call (0xE8), it needs to figure out
1558 * the jmp_offset relative to the jit-insn address immediately
1560 * the end of the jit-insn address after completely translated the
1561 * current (BPF_JMP | BPF_CALL) bpf-insn. It is passed as "end_addr"
1562 * to the emit_kfunc_call(). Thus, it can learn the "immediate-follow-call"
1563 * address by figuring out how many jit-insn is generated between
1565 * - 0-1 jit-insn (3 bytes each) to restore the esp pointer if there
1567 * - 0-2 jit-insns (3 bytes each) to handle the return value.
1583 return -EINVAL; in emit_kfunc_call()
1586 for (i = 0; i < fm->nr_args; i++) { in emit_kfunc_call()
1587 int regs_needed = fm->arg_size[i] > sizeof(u32) ? 2 : 1; in emit_kfunc_call()
1592 free_arg_regs -= regs_needed; in emit_kfunc_call()
1597 last_stack_regno = BPF_REG_0 + fm->nr_args; in emit_kfunc_call()
1598 for (i = last_stack_regno; i >= first_stack_regno; i--) { in emit_kfunc_call()
1599 if (fm->arg_size[i - 1] > sizeof(u32)) { in emit_kfunc_call()
1613 if (fm->arg_size[i - 1] > sizeof(u32)) in emit_kfunc_call()
1621 end_addr -= 3; in emit_kfunc_call()
1624 if (fm->ret_size > sizeof(u32)) in emit_kfunc_call()
1625 end_addr -= 3; in emit_kfunc_call()
1628 if (fm->ret_size) in emit_kfunc_call()
1629 end_addr -= 3; in emit_kfunc_call()
1631 jmp_offset = (u8 *)__bpf_call_base + insn->imm - end_addr; in emit_kfunc_call()
1635 return -EINVAL; in emit_kfunc_call()
1640 if (fm->ret_size) in emit_kfunc_call()
1645 if (fm->ret_size > sizeof(u32)) in emit_kfunc_call()
1662 struct bpf_insn *insn = bpf_prog->insnsi; in do_jit()
1663 int insn_cnt = bpf_prog->len; in do_jit()
1670 emit_prologue(&prog, bpf_prog->aux->stack_depth); in do_jit()
1673 const s32 imm32 = insn->imm; in do_jit()
1674 const bool is64 = BPF_CLASS(insn->code) == BPF_ALU64; in do_jit()
1675 const bool dstk = insn->dst_reg != BPF_REG_AX; in do_jit()
1676 const bool sstk = insn->src_reg != BPF_REG_AX; in do_jit()
1677 const u8 code = insn->code; in do_jit()
1678 const u8 *dst = bpf2ia32[insn->dst_reg]; in do_jit()
1679 const u8 *src = bpf2ia32[insn->src_reg]; in do_jit()
1701 &prog, bpf_prog->aux); in do_jit()
1704 /* Sign-extend immediate value to dst reg */ in do_jit()
1711 /* dst = dst - src/imm */ in do_jit()
1742 bpf_prog->aux); in do_jit()
1747 bpf_prog->aux); in do_jit()
1766 if (!bpf_prog->aux->verifier_zext) in do_jit()
1787 if (!bpf_prog->aux->verifier_zext) in do_jit()
1810 if (!bpf_prog->aux->verifier_zext) in do_jit()
1823 return -EINVAL; in do_jit()
1828 if (!bpf_prog->aux->verifier_zext) in do_jit()
1834 return -EINVAL; in do_jit()
1840 return -EINVAL; in do_jit()
1858 return -EINVAL; in do_jit()
1865 if (!bpf_prog->aux->verifier_zext) in do_jit()
1887 bpf_prog->aux); in do_jit()
1892 bpf_prog->aux); in do_jit()
1934 if (is_imm8(insn->off)) in do_jit()
1935 EMIT2(add_1reg(0x40, IA32_EAX), insn->off); in do_jit()
1938 insn->off); in do_jit()
1946 insn->off + 4); in do_jit()
1982 if (is_imm8(insn->off)) in do_jit()
1984 insn->off); in do_jit()
1987 insn->off); in do_jit()
2000 if (is_imm8(insn->off + 4)) { in do_jit()
2003 insn->off + 4); in do_jit()
2007 EMIT(insn->off + 4, 4); in do_jit()
2035 if (is_imm8(insn->off)) in do_jit()
2037 insn->off); in do_jit()
2040 insn->off); in do_jit()
2053 if (bpf_prog->aux->verifier_zext) in do_jit()
2068 insn->off + 4); in do_jit()
2091 if (insn->src_reg == BPF_PSEUDO_CALL) in do_jit()
2094 if (insn->src_reg == BPF_PSEUDO_KFUNC_CALL) { in do_jit()
2107 jmp_offset = func - (image + addrs[i]); in do_jit()
2112 return -EINVAL; in do_jit()
2141 emit_bpf_tail_call(&prog, image + addrs[i - 1]); in do_jit()
2161 bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP; in do_jit()
2232 bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP; in do_jit()
2276 bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP; in do_jit()
2332 bool is_jmp64 = BPF_CLASS(insn->code) == BPF_JMP; in do_jit()
2364 return -EFAULT; in do_jit()
2365 jmp_offset = addrs[i + insn->off] - addrs[i]; in do_jit()
2372 return -EFAULT; in do_jit()
2410 emit_cond_jmp_signed: /* Check the condition for low 32-bit comparison */ in do_jit()
2413 return -EFAULT; in do_jit()
2414 jmp_offset = addrs[i + insn->off] - addrs[i] + 8; in do_jit()
2419 return -EFAULT; in do_jit()
2423 /* Check the condition for high 32-bit comparison */ in do_jit()
2426 return -EFAULT; in do_jit()
2427 jmp_offset = addrs[i + insn->off] - addrs[i]; in do_jit()
2432 return -EFAULT; in do_jit()
2437 if (insn->off == -1) in do_jit()
2438 /* -1 jmp instructions will always jump in do_jit()
2444 jmp_offset = -2; in do_jit()
2446 jmp_offset = addrs[i + insn->off] - addrs[i]; in do_jit()
2458 return -EFAULT; in do_jit()
2466 jmp_offset = ctx->cleanup_addr - addrs[i]; in do_jit()
2471 ctx->cleanup_addr = proglen; in do_jit()
2472 emit_epilogue(&prog, bpf_prog->aux->stack_depth); in do_jit()
2476 return -EFAULT; in do_jit()
2484 return -EINVAL; in do_jit()
2487 ilen = prog - temp; in do_jit()
2490 return -EFAULT; in do_jit()
2505 return -EFAULT; in do_jit()
2533 if (!prog->jit_requested) in bpf_int_jit_compile()
2548 addrs = kmalloc_array(prog->len, sizeof(*addrs), GFP_KERNEL); in bpf_int_jit_compile()
2555 * Before first pass, make a rough estimation of addrs[] in bpf_int_jit_compile()
2558 for (proglen = 0, i = 0; i < prog->len; i++) { in bpf_int_jit_compile()
2601 bpf_jit_dump(prog->len, proglen, pass + 1, image); in bpf_int_jit_compile()
2604 prog->bpf_func = (void *)image; in bpf_int_jit_compile()
2605 prog->jited = 1; in bpf_int_jit_compile()
2606 prog->jited_len = proglen; in bpf_int_jit_compile()