riscv, bpf: Optimize bswap insns with Zbb support

Optimize bswap instructions by rev8 Zbb instruction conbined with srli
instruction. And Optimize 16-bit zero-extension with Zbb support.

Signed-off-by: Pu Lehui <pulehui@huawei.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Tested-by: Björn Töpel <bjorn@rivosinc.com>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20240115131235.2914289-7-pulehui@huaweicloud.com

diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index b00c5c0591d2046c9bc0c34cc76d0d5f82914aed..8b35f12a44527306e29348c5cf66e5f8e1bc1dc6 100644 (file)
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -1146,12 +1146,81 @@ static inline void emit_sextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
        emit_addiw(rd, rs, 0, ctx);
 }
 
+static inline void emit_zexth(u8 rd, u8 rs, struct rv_jit_context *ctx)
+{
+       if (rvzbb_enabled()) {
+               emit(rvzbb_zexth(rd, rs), ctx);
+               return;
+       }
+
+       emit_slli(rd, rs, 48, ctx);
+       emit_srli(rd, rd, 48, ctx);
+}
+
 static inline void emit_zextw(u8 rd, u8 rs, struct rv_jit_context *ctx)
 {
        emit_slli(rd, rs, 32, ctx);
        emit_srli(rd, rd, 32, ctx);
 }
 
+static inline void emit_bswap(u8 rd, s32 imm, struct rv_jit_context *ctx)
+{
+       if (rvzbb_enabled()) {
+               int bits = 64 - imm;
+
+               emit(rvzbb_rev8(rd, rd), ctx);
+               if (bits)
+                       emit_srli(rd, rd, bits, ctx);
+               return;
+       }
+
+       emit_li(RV_REG_T2, 0, ctx);
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+       if (imm == 16)
+               goto out_be;
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+       if (imm == 32)
+               goto out_be;
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+       emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
+       emit_srli(rd, rd, 8, ctx);
+out_be:
+       emit_andi(RV_REG_T1, rd, 0xff, ctx);
+       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
+
+       emit_mv(rd, RV_REG_T2, ctx);
+}
+
 #endif /* __riscv_xlen == 64 */
 
 void bpf_jit_build_prologue(struct rv_jit_context *ctx);

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index fc1a334e2f70d014b3e7b31f309351418ae75463..fda6b4f6a4c12015d197ee16d1d06305c8bd59c7 100644 (file)
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -1177,8 +1177,7 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
        case BPF_ALU | BPF_END | BPF_FROM_LE:
                switch (imm) {
                case 16:
-                       emit_slli(rd, rd, 48, ctx);
-                       emit_srli(rd, rd, 48, ctx);
+                       emit_zexth(rd, rd, ctx);
                        break;
                case 32:
                        if (!aux->verifier_zext)
@@ -1189,54 +1188,9 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
                        break;
                }
                break;
-
        case BPF_ALU | BPF_END | BPF_FROM_BE:
        case BPF_ALU64 | BPF_END | BPF_FROM_LE:
-               emit_li(RV_REG_T2, 0, ctx);
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-               if (imm == 16)
-                       goto out_be;
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-               if (imm == 32)
-                       goto out_be;
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-               emit_slli(RV_REG_T2, RV_REG_T2, 8, ctx);
-               emit_srli(rd, rd, 8, ctx);
-out_be:
-               emit_andi(RV_REG_T1, rd, 0xff, ctx);
-               emit_add(RV_REG_T2, RV_REG_T2, RV_REG_T1, ctx);
-
-               emit_mv(rd, RV_REG_T2, ctx);
+               emit_bswap(rd, imm, ctx);
                break;
 
        /* dst = imm */