x86_64: bpf: implement jitting of JMP32

This patch implements code-gen for new JMP32 instructions on x86_64.

Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Jiong Wang <jiong.wang@netronome.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c
index 5542303c43d9c525965135e8d8e3f6355c5ea032..afabf597c85577373a550539a5ae706859c5d61e 100644 (file)
--- a/arch/x86/net/bpf_jit_comp.c
+++ b/arch/x86/net/bpf_jit_comp.c
@@ -881,20 +881,41 @@ xadd:                     if (is_imm8(insn->off))
                case BPF_JMP | BPF_JSLT | BPF_X:
                case BPF_JMP | BPF_JSGE | BPF_X:
                case BPF_JMP | BPF_JSLE | BPF_X:
+               case BPF_JMP32 | BPF_JEQ | BPF_X:
+               case BPF_JMP32 | BPF_JNE | BPF_X:
+               case BPF_JMP32 | BPF_JGT | BPF_X:
+               case BPF_JMP32 | BPF_JLT | BPF_X:
+               case BPF_JMP32 | BPF_JGE | BPF_X:
+               case BPF_JMP32 | BPF_JLE | BPF_X:
+               case BPF_JMP32 | BPF_JSGT | BPF_X:
+               case BPF_JMP32 | BPF_JSLT | BPF_X:
+               case BPF_JMP32 | BPF_JSGE | BPF_X:
+               case BPF_JMP32 | BPF_JSLE | BPF_X:
                        /* cmp dst_reg, src_reg */
-                       EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x39,
-                             add_2reg(0xC0, dst_reg, src_reg));
+                       if (BPF_CLASS(insn->code) == BPF_JMP)
+                               EMIT1(add_2mod(0x48, dst_reg, src_reg));
+                       else if (is_ereg(dst_reg) || is_ereg(src_reg))
+                               EMIT1(add_2mod(0x40, dst_reg, src_reg));
+                       EMIT2(0x39, add_2reg(0xC0, dst_reg, src_reg));
                        goto emit_cond_jmp;
 
                case BPF_JMP | BPF_JSET | BPF_X:
+               case BPF_JMP32 | BPF_JSET | BPF_X:
                        /* test dst_reg, src_reg */
-                       EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x85,
-                             add_2reg(0xC0, dst_reg, src_reg));
+                       if (BPF_CLASS(insn->code) == BPF_JMP)
+                               EMIT1(add_2mod(0x48, dst_reg, src_reg));
+                       else if (is_ereg(dst_reg) || is_ereg(src_reg))
+                               EMIT1(add_2mod(0x40, dst_reg, src_reg));
+                       EMIT2(0x85, add_2reg(0xC0, dst_reg, src_reg));
                        goto emit_cond_jmp;
 
                case BPF_JMP | BPF_JSET | BPF_K:
+               case BPF_JMP32 | BPF_JSET | BPF_K:
                        /* test dst_reg, imm32 */
-                       EMIT1(add_1mod(0x48, dst_reg));
+                       if (BPF_CLASS(insn->code) == BPF_JMP)
+                               EMIT1(add_1mod(0x48, dst_reg));
+                       else if (is_ereg(dst_reg))
+                               EMIT1(add_1mod(0x40, dst_reg));
                        EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
                        goto emit_cond_jmp;
 
@@ -908,8 +929,21 @@ xadd:                      if (is_imm8(insn->off))
                case BPF_JMP | BPF_JSLT | BPF_K:
                case BPF_JMP | BPF_JSGE | BPF_K:
                case BPF_JMP | BPF_JSLE | BPF_K:
+               case BPF_JMP32 | BPF_JEQ | BPF_K:
+               case BPF_JMP32 | BPF_JNE | BPF_K:
+               case BPF_JMP32 | BPF_JGT | BPF_K:
+               case BPF_JMP32 | BPF_JLT | BPF_K:
+               case BPF_JMP32 | BPF_JGE | BPF_K:
+               case BPF_JMP32 | BPF_JLE | BPF_K:
+               case BPF_JMP32 | BPF_JSGT | BPF_K:
+               case BPF_JMP32 | BPF_JSLT | BPF_K:
+               case BPF_JMP32 | BPF_JSGE | BPF_K:
+               case BPF_JMP32 | BPF_JSLE | BPF_K:
                        /* cmp dst_reg, imm8/32 */
-                       EMIT1(add_1mod(0x48, dst_reg));
+                       if (BPF_CLASS(insn->code) == BPF_JMP)
+                               EMIT1(add_1mod(0x48, dst_reg));
+                       else if (is_ereg(dst_reg))
+                               EMIT1(add_1mod(0x40, dst_reg));
 
                        if (is_imm8(imm32))
                                EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);