riscv, bpf: Introduce shift add helper with Zba optimization

Zba extension is very useful for generating addresses that index into array
of basic data types. This patch introduces sh2add and sh3add helpers for
RV32 and RV64 respectively, to accelerate addressing for array of unsigned
long data.

Signed-off-by: Xiao Wang <xiao.w.wang@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Björn Töpel <bjorn@kernel.org>
Link: https://lore.kernel.org/bpf/20240524075543.4050464-3-xiao.w.wang@intel.com

diff --git a/arch/riscv/net/bpf_jit.h b/arch/riscv/net/bpf_jit.h
index 97041b58237a83d89f8cc90b95a9894efdd9f059..1d1c78d4cff1eee85eb244b5da7e3d933274857a 100644 (file)
--- a/arch/riscv/net/bpf_jit.h
+++ b/arch/riscv/net/bpf_jit.h
@@ -742,6 +742,17 @@ static inline u16 rvc_swsp(u32 imm8, u8 rs2)
        return rv_css_insn(0x6, imm, rs2, 0x2);
 }
 
+/* RVZBA instructions. */
+static inline u32 rvzba_sh2add(u8 rd, u8 rs1, u8 rs2)
+{
+       return rv_r_insn(0x10, rs2, rs1, 0x4, rd, 0x33);
+}
+
+static inline u32 rvzba_sh3add(u8 rd, u8 rs1, u8 rs2)
+{
+       return rv_r_insn(0x10, rs2, rs1, 0x6, rd, 0x33);
+}
+
 /* RVZBB instructions. */
 static inline u32 rvzbb_sextb(u8 rd, u8 rs1)
 {
@@ -1095,6 +1106,28 @@ static inline void emit_sw(u8 rs1, s32 off, u8 rs2, struct rv_jit_context *ctx)
                emit(rv_sw(rs1, off, rs2), ctx);
 }
 
+static inline void emit_sh2add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
+{
+       if (rvzba_enabled()) {
+               emit(rvzba_sh2add(rd, rs1, rs2), ctx);
+               return;
+       }
+
+       emit_slli(rd, rs1, 2, ctx);
+       emit_add(rd, rd, rs2, ctx);
+}
+
+static inline void emit_sh3add(u8 rd, u8 rs1, u8 rs2, struct rv_jit_context *ctx)
+{
+       if (rvzba_enabled()) {
+               emit(rvzba_sh3add(rd, rs1, rs2), ctx);
+               return;
+       }
+
+       emit_slli(rd, rs1, 3, ctx);
+       emit_add(rd, rd, rs2, ctx);
+}
+
 /* RV64-only helper functions. */
 #if __riscv_xlen == 64
 

diff --git a/arch/riscv/net/bpf_jit_comp32.c b/arch/riscv/net/bpf_jit_comp32.c
index f5ba73bb153d7ea0117eae54b379836bc0a0d8fc..592dd86fbf811afbe7d662a0e2b24d0541f02848 100644 (file)
--- a/arch/riscv/net/bpf_jit_comp32.c
+++ b/arch/riscv/net/bpf_jit_comp32.c
@@ -811,8 +811,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
         * if (!prog)
         *   goto out;
         */
-       emit(rv_slli(RV_REG_T0, lo(idx_reg), 2), ctx);
-       emit(rv_add(RV_REG_T0, RV_REG_T0, lo(arr_reg)), ctx);
+       emit_sh2add(RV_REG_T0, lo(idx_reg), lo(arr_reg), ctx);
        off = offsetof(struct bpf_array, ptrs);
        if (is_12b_check(off, insn))
                return -1;

diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c
index 2bc4c14ea59a5d77c8a3cb362f05738137eb6c0e..d5cebb0b0afeee7aa014b5a62c1c307521b296c5 100644 (file)
--- a/arch/riscv/net/bpf_jit_comp64.c
+++ b/arch/riscv/net/bpf_jit_comp64.c
@@ -380,8 +380,7 @@ static int emit_bpf_tail_call(int insn, struct rv_jit_context *ctx)
         * if (!prog)
         *     goto out;
         */
-       emit_slli(RV_REG_T2, RV_REG_A2, 3, ctx);
-       emit_add(RV_REG_T2, RV_REG_T2, RV_REG_A1, ctx);
+       emit_sh3add(RV_REG_T2, RV_REG_A2, RV_REG_A1, ctx);
        off = offsetof(struct bpf_array, ptrs);
        if (is_12b_check(off, insn))
                return -1;
@@ -1099,12 +1098,10 @@ int bpf_jit_emit_insn(const struct bpf_insn *insn, struct rv_jit_context *ctx,
                        /* Load current CPU number in T1 */
                        emit_ld(RV_REG_T1, offsetof(struct thread_info, cpu),
                                RV_REG_TP, ctx);
-                       /* << 3 because offsets are 8 bytes */
-                       emit_slli(RV_REG_T1, RV_REG_T1, 3, ctx);
                        /* Load address of __per_cpu_offset array in T2 */
                        emit_addr(RV_REG_T2, (u64)&__per_cpu_offset, extra_pass, ctx);
-                       /* Add offset of current CPU to  __per_cpu_offset */
-                       emit_add(RV_REG_T1, RV_REG_T2, RV_REG_T1, ctx);
+                       /* Get address of __per_cpu_offset[cpu] in T1 */
+                       emit_sh3add(RV_REG_T1, RV_REG_T1, RV_REG_T2, ctx);
                        /* Load __per_cpu_offset[cpu] in T1 */
                        emit_ld(RV_REG_T1, 0, RV_REG_T1, ctx);
                        /* Add the offset to Rd */