bpf: arm64: simplify exception table handling

BPF loads with BPF_PROBE_MEM(SX) can load from unsafe pointers and the
JIT adds an exception table entry for the JITed instruction which allows
the exeption handler to set the destination register of the load to zero
and continue execution from the next instruction.

As all arm64 instructions are AARCH64_INSN_SIZE size, the exception
handler can just increment the pc by AARCH64_INSN_SIZE without needing
the exact address of the instruction following the the faulting
instruction.

Simplify the exception table usage in arm64 JIT by only saving the
destination register in ex->fixup and drop everything related to
the fixup_offset. The fault handler is modified to add AARCH64_INSN_SIZE
to the pc.

Signed-off-by: Puranjay Mohan <puranjay@kernel.org>
Acked-by: Yonghong Song <yonghong.song@linux.dev>
Acked-by: Kumar Kartikeya Dwivedi <memxor@gmail.com>
Acked-by: Xu Kuohai <xukuohai@huawei.com>
Link: https://lore.kernel.org/r/20250911145808.58042-2-puranjay@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
index f0b1cb2c3bc4857220d3593ed25b4bfeaa0c43e2..e6d1fdc1e6f52a7061af71901d0c9b13b0461658 100644 (file)
--- a/arch/arm64/net/bpf_jit_comp.c
+++ b/arch/arm64/net/bpf_jit_comp.c
@@ -1066,19 +1066,18 @@ static void build_epilogue(struct jit_ctx *ctx, bool was_classic)
        emit(A64_RET(A64_LR), ctx);
 }
 
-#define BPF_FIXUP_OFFSET_MASK  GENMASK(26, 0)
 #define BPF_FIXUP_REG_MASK     GENMASK(31, 27)
 #define DONT_CLEAR 5 /* Unused ARM64 register from BPF's POV */
 
 bool ex_handler_bpf(const struct exception_table_entry *ex,
                    struct pt_regs *regs)
 {
-       off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup);
        int dst_reg = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup);
 
        if (dst_reg != DONT_CLEAR)
                regs->regs[dst_reg] = 0;
-       regs->pc = (unsigned long)&ex->fixup - offset;
+       /* Skip the faulting instruction */
+       regs->pc += AARCH64_INSN_SIZE;
        return true;
 }
 
@@ -1088,7 +1087,6 @@ static int add_exception_handler(const struct bpf_insn *insn,
                                 int dst_reg)
 {
        off_t ins_offset;
-       off_t fixup_offset;
        unsigned long pc;
        struct exception_table_entry *ex;
 
@@ -1119,22 +1117,6 @@ static int add_exception_handler(const struct bpf_insn *insn,
        if (WARN_ON_ONCE(ins_offset >= 0 || ins_offset < INT_MIN))
                return -ERANGE;
 
-       /*
-        * Since the extable follows the program, the fixup offset is always
-        * negative and limited to BPF_JIT_REGION_SIZE. Store a positive value
-        * to keep things simple, and put the destination register in the upper
-        * bits. We don't need to worry about buildtime or runtime sort
-        * modifying the upper bits because the table is already sorted, and
-        * isn't part of the main exception table.
-        *
-        * The fixup_offset is set to the next instruction from the instruction
-        * that may fault. The execution will jump to this after handling the
-        * fault.
-        */
-       fixup_offset = (long)&ex->fixup - (pc + AARCH64_INSN_SIZE);
-       if (!FIELD_FIT(BPF_FIXUP_OFFSET_MASK, fixup_offset))
-               return -ERANGE;
-
        /*
         * The offsets above have been calculated using the RO buffer but we
         * need to use the R/W buffer for writes.
@@ -1147,8 +1129,7 @@ static int add_exception_handler(const struct bpf_insn *insn,
        if (BPF_CLASS(insn->code) != BPF_LDX)
                dst_reg = DONT_CLEAR;
 
-       ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) |
-                   FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
+       ex->fixup = FIELD_PREP(BPF_FIXUP_REG_MASK, dst_reg);
 
        ex->type = EX_TYPE_BPF;