s390/bpf: Introduce pre- and post- probe functions

Currently probe insns are handled by two "if" statements at the
beginning and at the end of bpf_jit_insn(). The first one needs to be
in sync with the huge insn->code statement that follows it, which was
not a problem so far, since the check is small.

The introduction of arena will make it significantly larger, and it
will no longer be obvious whether it is in sync with the opcode switch.

Move these statements to the new bpf_jit_probe_load_pre() and
bpf_jit_probe_post() functions, and call them only from cases that need
them.

Signed-off-by: Ilya Leoshkevich <iii@linux.ibm.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Link: https://lore.kernel.org/bpf/20240701234304.14336-5-iii@linux.ibm.com

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index d9d79aa2be1ba14e3516555ef6fd16c44c28fc23..582fa38307725ab76adb8278b7c46ec2e81142bc 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -704,14 +704,28 @@ static void bpf_jit_probe_emit_nop(struct bpf_jit *jit,
        _EMIT2(0x0700);
 }
 
-static int bpf_jit_probe_mem(struct bpf_jit *jit, struct bpf_prog *fp,
-                            struct bpf_jit_probe *probe)
+static void bpf_jit_probe_load_pre(struct bpf_jit *jit, struct bpf_insn *insn,
+                                  struct bpf_jit_probe *probe)
+{
+       if (BPF_MODE(insn->code) != BPF_PROBE_MEM &&
+           BPF_MODE(insn->code) != BPF_PROBE_MEMSX)
+               return;
+
+       probe->prg = jit->prg;
+       probe->reg = reg2hex[insn->dst_reg];
+}
+
+static int bpf_jit_probe_post(struct bpf_jit *jit, struct bpf_prog *fp,
+                             struct bpf_jit_probe *probe)
 {
        struct exception_table_entry *ex;
        int i, prg;
        s64 delta;
        u8 *insn;
 
+       if (probe->prg == -1)
+               /* The probe is not armed. */
+               return 0;
        bpf_jit_probe_emit_nop(jit, probe);
        if (!fp->aux->extable)
                /* Do nothing during early JIT passes. */
@@ -798,12 +812,6 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
        int err;
 
        bpf_jit_probe_init(&probe);
-       if (BPF_CLASS(insn->code) == BPF_LDX &&
-           (BPF_MODE(insn->code) == BPF_PROBE_MEM ||
-            BPF_MODE(insn->code) == BPF_PROBE_MEMSX)) {
-               probe.prg = jit->prg;
-               probe.reg = reg2hex[dst_reg];
-       }
 
        switch (insn->code) {
        /*
@@ -1497,51 +1505,79 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp,
         */
        case BPF_LDX | BPF_MEM | BPF_B: /* dst = *(u8 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEM | BPF_B:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* llgc %dst,0(off,%src) */
                EMIT6_DISP_LH(0xe3000000, 0x0090, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                jit->seen |= SEEN_MEM;
                if (insn_is_zext(&insn[1]))
                        insn_count = 2;
                break;
        case BPF_LDX | BPF_MEMSX | BPF_B: /* dst = *(s8 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEMSX | BPF_B:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* lgb %dst,0(off,%src) */
                EMIT6_DISP_LH(0xe3000000, 0x0077, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                jit->seen |= SEEN_MEM;
                break;
        case BPF_LDX | BPF_MEM | BPF_H: /* dst = *(u16 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEM | BPF_H:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* llgh %dst,0(off,%src) */
                EMIT6_DISP_LH(0xe3000000, 0x0091, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                jit->seen |= SEEN_MEM;
                if (insn_is_zext(&insn[1]))
                        insn_count = 2;
                break;
        case BPF_LDX | BPF_MEMSX | BPF_H: /* dst = *(s16 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEMSX | BPF_H:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* lgh %dst,0(off,%src) */
                EMIT6_DISP_LH(0xe3000000, 0x0015, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                jit->seen |= SEEN_MEM;
                break;
        case BPF_LDX | BPF_MEM | BPF_W: /* dst = *(u32 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEM | BPF_W:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* llgf %dst,off(%src) */
                jit->seen |= SEEN_MEM;
                EMIT6_DISP_LH(0xe3000000, 0x0016, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                if (insn_is_zext(&insn[1]))
                        insn_count = 2;
                break;
        case BPF_LDX | BPF_MEMSX | BPF_W: /* dst = *(s32 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEMSX | BPF_W:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* lgf %dst,off(%src) */
                jit->seen |= SEEN_MEM;
                EMIT6_DISP_LH(0xe3000000, 0x0014, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                break;
        case BPF_LDX | BPF_MEM | BPF_DW: /* dst = *(u64 *)(ul) (src + off) */
        case BPF_LDX | BPF_PROBE_MEM | BPF_DW:
+               bpf_jit_probe_load_pre(jit, insn, &probe);
                /* lg %dst,0(off,%src) */
                jit->seen |= SEEN_MEM;
                EMIT6_DISP_LH(0xe3000000, 0x0004, dst_reg, src_reg, REG_0, off);
+               err = bpf_jit_probe_post(jit, fp, &probe);
+               if (err < 0)
+                       return err;
                break;
        /*
         * BPF_JMP / CALL
@@ -1906,12 +1942,6 @@ branch_oc:
                return -1;
        }
 
-       if (probe.prg != -1) {
-               err = bpf_jit_probe_mem(jit, fp, &probe);
-               if (err < 0)
-                       return err;
-       }
-
        return insn_count;
 }