Lines Matching +full:0 +full:x48
38 do { prog = emit_code(prog, bytes, len); } while (0)
46 do { EMIT1(b1); EMIT(off, 4); } while (0)
48 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
50 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
52 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
74 * 211: 74 7d je 0x290
75 * 213: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0]
78 * 28c: 74 17 je 0x2a5
79 * 28e: e9 7f ff ff ff jmp 0x212
80 * 293: bf 03 00 00 00 mov edi,0x3
81 * Note that insn at 0x211 is 2-byte cond jump insn for offset 0x7d (-125)
82 * and insn at 0x28e is 5-byte jmp insn with offset -129.
87 * 211: 0f 84 80 00 00 00 je 0x297
88 * 217: 48 8b 77 00 mov rsi,QWORD PTR [rdi+0x0]
91 * 290: 74 1a je 0x2ac
92 * 292: eb 84 jmp 0x218
93 * 294: bf 03 00 00 00 mov edi,0x3
94 * Note that insn at 0x211 is 6-byte cond jump insn now since its offset
95 * becomes 0x80 based on previous round (0x293 - 0x213 = 0x80).
96 * At the same time, insn at 0x292 is a 2-byte insn since its offset is
103 * cycle in the above. In the above example je offset <= 0x7c should work.
105 * For other cases, je <-> je needs offset <= 0x7b to avoid no convergence
106 * issue. For jmp <-> je and jmp <-> jmp cases, jmp offset <= 0x7c should
110 * to maximum 123 (0x7b). This way, the jit pass can eventually converge.
131 EMIT3(add_2mod(0x48, DST, SRC), 0x89, add_2reg(0xC0, DST, SRC)); \
132 } while (0)
145 return 0; in bpf_size_to_x86_bytes()
150 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
152 #define X86_JB 0x72
153 #define X86_JAE 0x73
154 #define X86_JE 0x74
155 #define X86_JNE 0x75
156 #define X86_JBE 0x76
157 #define X86_JA 0x77
158 #define X86_JL 0x7C
159 #define X86_JGE 0x7D
160 #define X86_JLE 0x7E
161 #define X86_JG 0x7F
179 [BPF_REG_0] = 0, /* RAX */
184 [BPF_REG_5] = 0, /* R8 */
286 [BPF_ADD] = 0x01,
287 [BPF_SUB] = 0x29,
288 [BPF_AND] = 0x21,
289 [BPF_OR] = 0x09,
290 [BPF_XOR] = 0x31,
291 [BPF_LSH] = 0xE0,
292 [BPF_RSH] = 0xE8,
293 [BPF_ARSH] = 0xF8,
299 memset(area, 0xcc, size); in jit_fill_hole()
304 return IS_ERR_OR_NULL(text_poke_set(dst, 0xcc, len)); in bpf_arch_text_invalidate()
332 EMIT2(0x41, 0x51); /* push r9 */ in push_r9()
340 EMIT2(0x41, 0x59); /* pop r9 */ in pop_r9()
348 EMIT2(0x41, 0x54); /* push r12 */ in push_r12()
356 if (callee_regs_used[0]) in push_callee_regs()
357 EMIT1(0x53); /* push rbx */ in push_callee_regs()
359 EMIT2(0x41, 0x55); /* push r13 */ in push_callee_regs()
361 EMIT2(0x41, 0x56); /* push r14 */ in push_callee_regs()
363 EMIT2(0x41, 0x57); /* push r15 */ in push_callee_regs()
371 EMIT2(0x41, 0x5C); /* pop r12 */ in pop_r12()
380 EMIT2(0x41, 0x5F); /* pop r15 */ in pop_callee_regs()
382 EMIT2(0x41, 0x5E); /* pop r14 */ in pop_callee_regs()
384 EMIT2(0x41, 0x5D); /* pop r13 */ in pop_callee_regs()
385 if (callee_regs_used[0]) in pop_callee_regs()
386 EMIT1(0x5B); /* pop rbx */ in pop_callee_regs()
395 while (len > 0) { in emit_nops()
401 for (i = 0; i < noplen; i++) in emit_nops()
419 EMIT3_off32(0x41, 0x81, 0xea, hash); /* subl $hash, %r10d */ in emit_fineibt()
420 EMIT2(0x74, 0x07); /* jz.d8 +7 */ in emit_fineibt()
421 EMIT2(0x0f, 0x0b); /* ud2 */ in emit_fineibt()
422 EMIT1(0x90); /* nop */ in emit_fineibt()
432 EMIT1_off32(0xb8, hash); /* movl $hash, %eax */ in emit_kcfi()
434 EMIT1(0x90); in emit_kcfi()
435 EMIT1(0x90); in emit_kcfi()
436 EMIT1(0x90); in emit_kcfi()
437 EMIT1(0x90); in emit_kcfi()
438 EMIT1(0x90); in emit_kcfi()
439 EMIT1(0x90); in emit_kcfi()
440 EMIT1(0x90); in emit_kcfi()
441 EMIT1(0x90); in emit_kcfi()
442 EMIT1(0x90); in emit_kcfi()
443 EMIT1(0x90); in emit_kcfi()
444 EMIT1(0x90); in emit_kcfi()
478 EMIT4(0x48, 0x83, 0xF8, MAX_TAIL_CALL_CNT); in emit_prologue_tail_call()
484 EMIT1(0x50); /* push rax */ in emit_prologue_tail_call()
486 EMIT3(0x48, 0x89, 0xE0); /* mov rax, rsp */ in emit_prologue_tail_call()
487 EMIT2(0xEB, 1); /* jmp 1 */ in emit_prologue_tail_call()
491 EMIT1(0x50); /* push rax */ in emit_prologue_tail_call()
493 EMIT1(0x50); /* push rax */ in emit_prologue_tail_call()
496 EMIT1(0x50); /* push rax */ in emit_prologue_tail_call()
497 EMIT1(0x50); /* push rax */ in emit_prologue_tail_call()
524 EMIT3(0x48, 0x31, 0xC0); /* xor rax, rax */ in emit_prologue()
531 EMIT3(0x48, 0x89, 0xF4); /* mov rsp, rsi */ in emit_prologue()
532 EMIT3(0x48, 0x89, 0xD5); /* mov rbp, rdx */ in emit_prologue()
540 EMIT3(0x48, 0x89, 0xEC); /* mov rsp, rbp */ in emit_prologue()
542 EMIT1(0x55); /* push rbp */ in emit_prologue()
543 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in emit_prologue()
551 EMIT3_off32(0x48, 0x81, 0xEC, round_up(stack_depth, 8)); in emit_prologue()
569 return 0; in emit_patch()
574 return emit_patch(pprog, func, ip, 0xE8); in emit_call()
581 return emit_patch(pprog, func, ip, 0xE8); in emit_rsb_call()
586 return emit_patch(pprog, func, ip, 0xE9); in emit_jump()
625 ret = 0; in __bpf_arch_text_poke()
650 #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8)
658 EMIT2(0xFF, 0xE0 + reg); in emit_indirect_jump()
666 EMIT2(0xFF, 0xE0 + reg); /* jmp *%\reg */ in emit_indirect_jump()
668 EMIT1(0xCC); /* int3 */ in emit_indirect_jump()
681 EMIT1(0xC3); /* ret */ in emit_return()
683 EMIT1(0xCC); /* int3 */ in emit_return()
724 EMIT2(0x89, 0xD2); /* mov edx, edx */ in emit_bpf_tail_call_indirect()
725 EMIT3(0x39, 0x56, /* cmp dword ptr [rsi + 16], edx */ in emit_bpf_tail_call_indirect()
735 EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ in emit_bpf_tail_call_indirect()
736 EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_indirect()
742 EMIT4_off32(0x48, 0x8B, 0x8C, 0xD6, /* mov rcx, [rsi + rdx * 8 + offsetof(...)] */ in emit_bpf_tail_call_indirect()
749 EMIT3(0x48, 0x85, 0xC9); /* test rcx,rcx */ in emit_bpf_tail_call_indirect()
755 EMIT4(0x48, 0x83, 0x00, 0x01); /* add qword ptr [rax], 1 */ in emit_bpf_tail_call_indirect()
767 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_indirect()
771 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_indirect()
773 EMIT3_off32(0x48, 0x81, 0xC4, /* add rsp, sd */ in emit_bpf_tail_call_indirect()
777 EMIT4(0x48, 0x8B, 0x49, /* mov rcx, qword ptr [rcx + 32] */ in emit_bpf_tail_call_indirect()
779 EMIT4(0x48, 0x83, 0xC1, /* add rcx, X86_TAIL_CALL_OFFSET */ in emit_bpf_tail_call_indirect()
807 EMIT3_off32(0x48, 0x8B, 0x85, tcc_ptr_off); /* mov rax, qword ptr [rbp - tcc_ptr_off] */ in emit_bpf_tail_call_direct()
808 EMIT4(0x48, 0x83, 0x38, MAX_TAIL_CALL_CNT); /* cmp qword ptr [rax], MAX_TAIL_CALL_CNT */ in emit_bpf_tail_call_direct()
822 EMIT4(0x48, 0x83, 0x00, 0x01); /* add qword ptr [rax], 1 */ in emit_bpf_tail_call_direct()
834 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_direct()
838 EMIT1(0x58); /* pop rax */ in emit_bpf_tail_call_direct()
840 EMIT3_off32(0x48, 0x81, 0xC4, round_up(stack_depth, 8)); in emit_bpf_tail_call_direct()
857 for (i = 0; i < prog->aux->size_poke_tab; i++) { in bpf_tail_call_direct_fixup()
875 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
880 BUG_ON(ret < 0); in bpf_tail_call_direct_fixup()
897 if (sign_propagate && (s32)imm32 < 0) { in emit_mov_imm32()
899 b1 = add_1mod(0x48, dst_reg); in emit_mov_imm32()
900 b2 = 0xC7; in emit_mov_imm32()
901 b3 = 0xC0; in emit_mov_imm32()
910 if (imm32 == 0) { in emit_mov_imm32()
912 EMIT1(add_2mod(0x40, dst_reg, dst_reg)); in emit_mov_imm32()
913 b2 = 0x31; /* xor */ in emit_mov_imm32()
914 b3 = 0xC0; in emit_mov_imm32()
921 EMIT1(add_1mod(0x40, dst_reg)); in emit_mov_imm32()
922 EMIT1_off32(add_1reg(0xB8, dst_reg), imm32); in emit_mov_imm32()
945 EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg)); in emit_mov_imm64()
963 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in emit_mov_reg()
964 EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg)); in emit_mov_reg()
978 EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbe, in emit_movsx_reg()
979 add_2reg(0xC0, src_reg, dst_reg)); in emit_movsx_reg()
981 EMIT4(add_2mod(0x48, src_reg, dst_reg), 0x0f, 0xbf, in emit_movsx_reg()
982 add_2reg(0xC0, src_reg, dst_reg)); in emit_movsx_reg()
984 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x63, in emit_movsx_reg()
985 add_2reg(0xC0, src_reg, dst_reg)); in emit_movsx_reg()
989 EMIT4(add_2mod(0x40, src_reg, dst_reg), 0x0f, 0xbe, in emit_movsx_reg()
990 add_2reg(0xC0, src_reg, dst_reg)); in emit_movsx_reg()
993 EMIT1(add_2mod(0x40, src_reg, dst_reg)); in emit_movsx_reg()
994 EMIT3(add_2mod(0x0f, src_reg, dst_reg), 0xbf, in emit_movsx_reg()
995 add_2reg(0xC0, src_reg, dst_reg)); in emit_movsx_reg()
1010 * If off == 0 we could skip this and save one extra byte, but in emit_insn_suffix()
1014 EMIT2(add_2reg(0x40, ptr_reg, val_reg), off); in emit_insn_suffix()
1017 EMIT1_off32(add_2reg(0x80, ptr_reg, val_reg), off); in emit_insn_suffix()
1027 EMIT3(add_2reg(0x44, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); in emit_insn_suffix_SIB()
1029 EMIT2_off32(add_2reg(0x84, BPF_REG_0, val_reg), add_2reg(0, ptr_reg, index_reg) /* SIB */, off); in emit_insn_suffix_SIB()
1042 EMIT1(add_2mod(0x48, dst_reg, src_reg)); in maybe_emit_mod()
1044 EMIT1(add_2mod(0x40, dst_reg, src_reg)); in maybe_emit_mod()
1056 EMIT1(add_1mod(0x48, reg)); in maybe_emit_1mod()
1058 EMIT1(add_1mod(0x40, reg)); in maybe_emit_1mod()
1070 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6); in emit_ldx()
1074 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7); in emit_ldx()
1077 /* Emit 'mov eax, dword ptr [rax+0x14]' */ in emit_ldx()
1079 EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B); in emit_ldx()
1081 EMIT1(0x8B); in emit_ldx()
1084 /* Emit 'mov rax, qword ptr [rax+0x14]' */ in emit_ldx()
1085 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B); in emit_ldx()
1100 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBE); in emit_ldsx()
1104 EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xBF); in emit_ldsx()
1107 /* Emit 'movsx rax, dword ptr [rax+0x14]' */ in emit_ldsx()
1108 EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x63); in emit_ldsx()
1122 EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB6); in emit_ldx_index()
1126 EMIT3(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x0F, 0xB7); in emit_ldx_index()
1130 EMIT2(add_3mod(0x40, src_reg, dst_reg, index_reg), 0x8B); in emit_ldx_index()
1134 EMIT2(add_3mod(0x48, src_reg, dst_reg, index_reg), 0x8B); in emit_ldx_index()
1156 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88); in emit_stx()
1158 EMIT1(0x88); in emit_stx()
1162 EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
1164 EMIT2(0x66, 0x89); in emit_stx()
1168 EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89); in emit_stx()
1170 EMIT1(0x89); in emit_stx()
1173 EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89); in emit_stx()
1188 EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x88); in emit_stx_index()
1192 EMIT3(0x66, add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89); in emit_stx_index()
1196 EMIT2(add_3mod(0x40, dst_reg, src_reg, index_reg), 0x89); in emit_stx_index()
1200 EMIT2(add_3mod(0x48, dst_reg, src_reg, index_reg), 0x89); in emit_stx_index()
1220 EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC6); in emit_st_index()
1224 EMIT3(0x66, add_3mod(0x40, dst_reg, 0, index_reg), 0xC7); in emit_st_index()
1228 EMIT2(add_3mod(0x40, dst_reg, 0, index_reg), 0xC7); in emit_st_index()
1232 EMIT2(add_3mod(0x48, dst_reg, 0, index_reg), 0xC7); in emit_st_index()
1235 emit_insn_suffix_SIB(&prog, dst_reg, 0, index_reg, off); in emit_st_index()
1250 EMIT1(0xF0); /* lock prefix */ in emit_atomic()
1265 EMIT2(0x0F, 0xC1); in emit_atomic()
1269 EMIT1(0x87); in emit_atomic()
1273 EMIT2(0x0F, 0xB1); in emit_atomic()
1283 return 0; in emit_atomic()
1291 EMIT1(0xF0); /* lock prefix */ in emit_atomic_index()
1294 EMIT1(add_3mod(0x40, dst_reg, src_reg, index_reg)); in emit_atomic_index()
1297 EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg)); in emit_atomic_index()
1315 EMIT2(0x0F, 0xC1); in emit_atomic_index()
1319 EMIT1(0x87); in emit_atomic_index()
1323 EMIT2(0x0F, 0xB1); in emit_atomic_index()
1331 return 0; in emit_atomic_index()
1342 *(unsigned long *)((void *)regs + reg) = 0; in ex_handler_bpf()
1343 regs->ip += x->fixup & 0xff; in ex_handler_bpf()
1354 regs_used[0] = true; in detect_reg_usage()
1369 * m: opcode map select, encoding escape bytes e.g. 0x0f38
1373 * pp: opcode prefix (none, 0x66, 0xf2 or 0xf3)
1379 const u8 b0 = 0xc4; /* first byte of 3-byte VEX prefix */ in emit_3vex()
1391 * 7 0 in emit_3vex()
1396 b1 = (!r << 7) | (!x << 6) | (!b << 5) | (m & 0x1f); in emit_3vex()
1400 * 7 0 in emit_3vex()
1405 b2 = (w << 7) | ((~vvvv & 0xf) << 3) | (l << 2) | (pp & 3); in emit_3vex()
1416 u8 m = 2; /* escape code 0f38 */ in emit_shiftx()
1419 EMIT2(0xf7, add_2reg(0xC0, dst_reg, dst_reg)); in emit_shiftx()
1433 EMIT2(0x65, 0x4c); in emit_priv_frame_ptr()
1434 EMIT3(0x03, 0x0c, 0x25); in emit_priv_frame_ptr()
1444 EMIT3_off32(0x48, 0x8B, 0x85, off)
1451 #define PRIV_STACK_GUARD_VAL 0xEB9F12345678eb9fULL
1465 int i, excnt = 0; in do_jit()
1466 int ilen, proglen = 0; in do_jit()
1475 stack_depth = 0; in do_jit()
1512 addrs[0] = proglen; in do_jit()
1519 u8 b2 = 0, b3 = 0; in do_jit()
1550 EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
1560 EMIT3(0xC1, add_1reg(0xE0, dst_reg), 32); in do_jit()
1565 EMIT1_off32(0x0D, user_vm_start >> 32); in do_jit()
1567 EMIT2_off32(0x81, add_1reg(0xC8, dst_reg), user_vm_start >> 32); in do_jit()
1571 EMIT3(0xC1, add_1reg(0xC0, dst_reg), 32); in do_jit()
1574 EMIT3(0x4D, 0x31, 0xDB); in do_jit()
1578 EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1583 EMIT3(0x0F, 0x44, add_2reg(0xC0, AUX_REG, dst_reg)); in do_jit()
1590 EMIT2(0x65, add_1mod(0x48, dst_reg)); in do_jit()
1591 EMIT3(0x03, add_2reg(0x04, 0, dst_reg), 0x25); in do_jit()
1598 if (insn->off == 0) in do_jit()
1613 EMIT2(0xF7, add_1reg(0xD8, dst_reg)); in do_jit()
1635 b3 = 0xC0; in do_jit()
1636 b2 = 0x05; in do_jit()
1639 b3 = 0xE8; in do_jit()
1640 b2 = 0x2D; in do_jit()
1643 b3 = 0xE0; in do_jit()
1644 b2 = 0x25; in do_jit()
1647 b3 = 0xC8; in do_jit()
1648 b2 = 0x0D; in do_jit()
1651 b3 = 0xF0; in do_jit()
1652 b2 = 0x35; in do_jit()
1657 EMIT3(0x83, add_1reg(b3, dst_reg), imm32); in do_jit()
1661 EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32); in do_jit()
1671 emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm); in do_jit()
1688 EMIT1(0x50); /* push rax */ in do_jit()
1690 EMIT1(0x52); /* push rdx */ in do_jit()
1701 EMIT3_off32(0x49, 0xC7, 0xC3, imm32); in do_jit()
1709 if (insn->off == 0) { in do_jit()
1714 EMIT2(0x31, 0xd2); in do_jit()
1718 EMIT2(0xF7, add_1reg(0xF0, src_reg)); in do_jit()
1721 EMIT1(0x99); /* cdq */ in do_jit()
1723 EMIT2(0x48, 0x99); /* cqo */ in do_jit()
1727 EMIT2(0xF7, add_1reg(0xF8, src_reg)); in do_jit()
1740 EMIT1(0x5A); /* pop rdx */ in do_jit()
1742 EMIT1(0x58); /* pop rax */ in do_jit()
1753 EMIT3(0x6B, add_2reg(0xC0, dst_reg, dst_reg), in do_jit()
1757 EMIT2_off32(0x69, in do_jit()
1758 add_2reg(0xC0, dst_reg, dst_reg), in do_jit()
1768 EMIT3(0x0F, 0xAF, add_2reg(0xC0, src_reg, dst_reg)); in do_jit()
1783 EMIT2(0xD1, add_1reg(b3, dst_reg)); in do_jit()
1785 EMIT3(0xC1, add_1reg(b3, dst_reg), imm32); in do_jit()
1802 op = 1; /* prefix 0x66 */ in do_jit()
1805 op = 3; /* prefix 0xf2 */ in do_jit()
1808 op = 2; /* prefix 0xf3 */ in do_jit()
1824 EMIT1(0x51); /* push rcx */ in do_jit()
1835 EMIT2(0xD3, add_1reg(b3, dst_reg)); in do_jit()
1842 EMIT1(0x59); /* pop rcx */ in do_jit()
1852 EMIT1(0x66); in do_jit()
1854 EMIT1(0x41); in do_jit()
1855 EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8); in do_jit()
1859 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1861 EMIT2(0x0F, 0xB7); in do_jit()
1862 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1867 EMIT2(0x41, 0x0F); in do_jit()
1869 EMIT1(0x0F); in do_jit()
1870 EMIT1(add_1reg(0xC8, dst_reg)); in do_jit()
1874 EMIT3(add_1mod(0x48, dst_reg), 0x0F, in do_jit()
1875 add_1reg(0xC8, dst_reg)); in do_jit()
1888 EMIT3(0x45, 0x0F, 0xB7); in do_jit()
1890 EMIT2(0x0F, 0xB7); in do_jit()
1891 EMIT1(add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1896 EMIT1(0x45); in do_jit()
1897 EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
1913 EMIT2(0x41, 0xC6); in do_jit()
1915 EMIT1(0xC6); in do_jit()
1919 EMIT3(0x66, 0x41, 0xC7); in do_jit()
1921 EMIT2(0x66, 0xC7); in do_jit()
1925 EMIT2(0x41, 0xC7); in do_jit()
1927 EMIT1(0xC7); in do_jit()
1930 EMIT2(add_1mod(0x48, dst_reg), 0xC7); in do_jit()
1933 EMIT2(add_1reg(0x40, dst_reg), insn->off); in do_jit()
1935 EMIT1_off32(add_1reg(0x80, dst_reg), insn->off); in do_jit()
2037 EMIT2_off32(0x81, add_1reg(0xC0, AUX_REG), insn->off); in do_jit()
2042 EMIT2(0x29, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); in do_jit()
2050 EMIT2(0x39, add_2reg(0xC0, AUX_REG, BPF_REG_AX)); in do_jit()
2053 EMIT2(X86_JA, 0); in do_jit()
2057 emit_mov_imm32(&prog, false, dst_reg, 0); in do_jit()
2059 EMIT2(0xEB, 0); in do_jit()
2109 * End result: x86 insn "mov rbx, qword ptr [rax+0x14]" in do_jit()
2149 add_2reg(0xC0, AUX_REG, real_src_reg)); in do_jit()
2248 EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
2256 EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg)); in do_jit()
2264 EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32); in do_jit()
2288 if (imm32 == 0) { in do_jit()
2291 EMIT2(0x85, add_2reg(0xC0, dst_reg, dst_reg)); in do_jit()
2300 EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32); in do_jit()
2302 EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32); in do_jit()
2357 * "nops" is 0. in do_jit()
2367 if (nops != 0 && nops != 4) { in do_jit()
2376 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2413 * to pad any extra byte (0 byte). in do_jit()
2421 if (nops != 0 && nops != 2 && nops != 5) { in do_jit()
2438 * jmp, there is nothing to pad (0 byte). in do_jit()
2446 if (nops != 0 && nops != 3) { in do_jit()
2453 EMIT2(0xEB, jmp_offset); in do_jit()
2455 EMIT1_off32(0xE9, jmp_offset); in do_jit()
2478 EMIT1(0xC9); /* leave */ in do_jit()
2565 /* mov DWORD PTR [rbp + off], 0 */ in clean_stack_garbage()
2567 EMIT2_off32(0xC7, 0x85, off); in clean_stack_garbage()
2569 EMIT3(0xC7, 0x45, off); in clean_stack_garbage()
2570 EMIT(0, 4); in clean_stack_garbage()
2578 int i, arg_regs, nr_used_regs = 0; in get_nr_used_regs()
2580 for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { in get_nr_used_regs()
2595 int arg_regs, first_off = 0, nr_regs = 0, nr_stack_slots = 0; in save_args()
2600 * mov QWORD PTR [rbp-0x10],rdi in save_args()
2601 * mov QWORD PTR [rbp-0x8],rsi in save_args()
2603 for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { in save_args()
2634 for (j = 0; j < arg_regs; j++) { in save_args()
2636 nr_stack_slots * 8 + 0x18); in save_args()
2656 for (j = 0; j < arg_regs; j++) { in save_args()
2672 int i, j, arg_regs, nr_regs = 0; in restore_regs()
2676 * EMIT4(0x48, 0x8B, 0x7D, 0xF0); mov rdi,QWORD PTR [rbp-0x10] in restore_regs()
2677 * EMIT4(0x48, 0x8B, 0x75, 0xF8); mov rsi,QWORD PTR [rbp-0x8] in restore_regs()
2681 for (i = 0; i < min_t(int, m->nr_args, MAX_BPF_FUNC_ARGS); i++) { in restore_regs()
2684 for (j = 0; j < arg_regs; j++) { in restore_regs()
2728 EMIT3_off32(0x48, 0x8D, 0xB5, -run_ctx_off); in invoke_bpf_prog()
2730 EMIT4(0x48, 0x8D, 0x75, -run_ctx_off); in invoke_bpf_prog()
2737 /* if (__bpf_prog_enter*(prog) == 0) in invoke_bpf_prog()
2740 EMIT3(0x48, 0x85, 0xC0); /* test rax,rax */ in invoke_bpf_prog()
2747 EMIT3_off32(0x48, 0x8D, 0xBD, -stack_size); in invoke_bpf_prog()
2749 EMIT4(0x48, 0x8D, 0x7D, -stack_size); in invoke_bpf_prog()
2771 jmp_insn[0] = X86_JE; in invoke_bpf_prog()
2780 EMIT3_off32(0x48, 0x8D, 0x95, -run_ctx_off); in invoke_bpf_prog()
2782 EMIT4(0x48, 0x8D, 0x55, -run_ctx_off); in invoke_bpf_prog()
2787 return 0; in invoke_bpf_prog()
2811 EMIT2_off32(0x0F, jmp_cond + 0x10, offset); in emit_cond_near_jump()
2813 return 0; in emit_cond_near_jump()
2824 for (i = 0; i < tl->nr_links; i++) { in invoke_bpf()
2830 return 0; in invoke_bpf()
2842 * Set this to 0 to avoid confusing the program. in invoke_bpf_mod_ret()
2844 emit_mov_imm32(&prog, false, BPF_REG_0, 0); in invoke_bpf_mod_ret()
2846 for (i = 0; i < tl->nr_links; i++) { in invoke_bpf_mod_ret()
2852 * if (*(u64 *)(rbp - 8) != 0) in invoke_bpf_mod_ret()
2855 /* cmp QWORD PTR [rbp - 0x8], 0x0 */ in invoke_bpf_mod_ret()
2856 EMIT4(0x48, 0x83, 0x7d, 0xf8); EMIT1(0x00); in invoke_bpf_mod_ret()
2868 return 0; in invoke_bpf_mod_ret()
2941 int i, ret, nr_regs = m->nr_args, stack_size = 0; in __arch_prepare_bpf_trampoline()
2960 for (i = 0; i < m->nr_args; i++) { in __arch_prepare_bpf_trampoline()
2974 * RBP + 0 [ RBP ] in __arch_prepare_bpf_trampoline()
3018 stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7; in __arch_prepare_bpf_trampoline()
3030 stack_size += (stack_size % 16) ? 0 : 8; in __arch_prepare_bpf_trampoline()
3058 EMIT1(0x55); /* push rbp */ in __arch_prepare_bpf_trampoline()
3059 EMIT3(0x48, 0x89, 0xE5); /* mov rbp, rsp */ in __arch_prepare_bpf_trampoline()
3062 EMIT3_off32(0x48, 0x81, 0xEC, stack_size); in __arch_prepare_bpf_trampoline()
3065 EMIT4(0x48, 0x83, 0xEC, stack_size); in __arch_prepare_bpf_trampoline()
3068 EMIT1(0x50); /* push rax */ in __arch_prepare_bpf_trampoline()
3076 emit_mov_imm64(&prog, BPF_REG_0, 0, (u32) nr_regs); in __arch_prepare_bpf_trampoline()
3132 EMIT2(0xff, 0xd3); /* call *rbx */ in __arch_prepare_bpf_trampoline()
3156 for (i = 0; i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
3197 EMIT1(0xC9); /* leave */ in __arch_prepare_bpf_trampoline()
3200 EMIT4(0x48, 0x83, 0xC4, 8); /* add rsp, 8 */ in __arch_prepare_bpf_trampoline()
3227 return 0; in arch_protect_bpf_trampoline()
3248 if (ret < 0) in arch_prepare_bpf_trampoline()
3293 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
3296 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), in emit_bpf_dispatcher()
3307 return 0; in emit_bpf_dispatcher()
3314 EMIT1(add_1mod(0x48, BPF_REG_3)); /* cmp rdx,func */ in emit_bpf_dispatcher()
3317 EMIT2_off32(0x81, add_1reg(0xF8, BPF_REG_3), progs[a + pivot]); in emit_bpf_dispatcher()
3322 EMIT2_off32(0x0F, X86_JG + 0x10, 0); in emit_bpf_dispatcher()
3324 EMIT2(X86_JG, 0); in emit_bpf_dispatcher()
3348 return 0; in emit_bpf_dispatcher()
3360 return 0; in cmp_ips()
3367 sort(funcs, num_funcs, sizeof(funcs[0]), cmp_ips, NULL); in arch_prepare_bpf_dispatcher()
3368 return emit_bpf_dispatcher(&prog, 0, num_funcs - 1, funcs, image, buf); in arch_prepare_bpf_dispatcher()
3385 stack_ptr[0] = PRIV_STACK_GUARD_VAL; in priv_stack_init_guard()
3398 if (stack_ptr[0] != PRIV_STACK_GUARD_VAL || in priv_stack_check_guard()
3427 int proglen, oldproglen = 0; in bpf_int_jit_compile()
3501 for (proglen = 0, i = 0; i <= prog->len; i++) { in bpf_int_jit_compile()
3514 for (pass = 0; pass < MAX_PASSES || image; pass++) { in bpf_int_jit_compile()
3518 if (proglen <= 0) { in bpf_int_jit_compile()
3530 prog->jited = 0; in bpf_int_jit_compile()
3531 prog->jited_len = 0; in bpf_int_jit_compile()
3737 BUG_ON(ret < 0); in bpf_arch_poke_desc_update()
3743 BUG_ON(ret < 0); in bpf_arch_poke_desc_update()
3750 BUG_ON(ret < 0); in bpf_arch_poke_desc_update()
3760 BUG_ON(ret < 0); in bpf_arch_poke_desc_update()
3789 /* x86-64 JIT emits its own code to filter user addresses so return 0 here */
3792 return 0; in bpf_arch_uaddress_limit()