bpf, s390: add support for constant blinding

This patch adds recently added constant blinding helpers into the
s390 eBPF JIT. In the bpf_int_jit_compile() path, requirements are
to utilize bpf_jit_blind_constants()/bpf_jit_prog_release_other()
pair for rewriting the program into a blinded one, and to map the
BPF_REG_AX register to a CPU register. The mapping of BPF_REG_AX
is at r12 and similarly like in x86 case performs reloading when
ld_abs/ind is used. When blinding is not used, there's no additional
overhead in the generated image.

When BPF_REG_AX is used, we don't need to emit skb->data reload when
helper function changed skb->data, as this will be reloaded later
on anyway from stack on ld_abs/ind, where skb->data is needed. s390
allows for this w/o much additional complexity unlike f.e. x86.

Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Signed-off-by: Michael Holzheu <holzheu@linux.vnet.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>

diff --git a/arch/s390/net/bpf_jit_comp.c b/arch/s390/net/bpf_jit_comp.c
index fcf301a889e7dd3a2e3d6668c943cfe28c94cdb6..9133b0ec000b82037729ac8ff7e4912b252b13a2 100644 (file)
--- a/arch/s390/net/bpf_jit_comp.c
+++ b/arch/s390/net/bpf_jit_comp.c
@@ -54,16 +54,17 @@ struct bpf_jit {
 #define SEEN_FUNC      16      /* calls C functions */
 #define SEEN_TAIL_CALL 32      /* code uses tail calls */
 #define SEEN_SKB_CHANGE        64      /* code changes skb data */
+#define SEEN_REG_AX    128     /* code uses constant blinding */
 #define SEEN_STACK     (SEEN_FUNC | SEEN_MEM | SEEN_SKB)
 
 /*
  * s390 registers
  */
-#define REG_W0         (__MAX_BPF_REG+0)       /* Work register 1 (even) */
-#define REG_W1         (__MAX_BPF_REG+1)       /* Work register 2 (odd) */
-#define REG_SKB_DATA   (__MAX_BPF_REG+2)       /* SKB data register */
-#define REG_L          (__MAX_BPF_REG+3)       /* Literal pool register */
-#define REG_15         (__MAX_BPF_REG+4)       /* Register 15 */
+#define REG_W0         (MAX_BPF_JIT_REG + 0)   /* Work register 1 (even) */
+#define REG_W1         (MAX_BPF_JIT_REG + 1)   /* Work register 2 (odd) */
+#define REG_SKB_DATA   (MAX_BPF_JIT_REG + 2)   /* SKB data register */
+#define REG_L          (MAX_BPF_JIT_REG + 3)   /* Literal pool register */
+#define REG_15         (MAX_BPF_JIT_REG + 4)   /* Register 15 */
 #define REG_0          REG_W0                  /* Register 0 */
 #define REG_1          REG_W1                  /* Register 1 */
 #define REG_2          BPF_REG_1               /* Register 2 */
@@ -88,6 +89,8 @@ static const int reg2hex[] = {
        [BPF_REG_9]     = 10,
        /* BPF stack pointer */
        [BPF_REG_FP]    = 13,
+       /* Register for blinding (shared with REG_SKB_DATA) */
+       [BPF_REG_AX]    = 12,
        /* SKB data pointer */
        [REG_SKB_DATA]  = 12,
        /* Work registers for s390x backend */
@@ -385,7 +388,7 @@ static void save_restore_regs(struct bpf_jit *jit, int op)
 /*
  * For SKB access %b1 contains the SKB pointer. For "bpf_jit.S"
  * we store the SKB header length on the stack and the SKB data
- * pointer in REG_SKB_DATA.
+ * pointer in REG_SKB_DATA if BPF_REG_AX is not used.
  */
 static void emit_load_skb_data_hlen(struct bpf_jit *jit)
 {
@@ -397,9 +400,10 @@ static void emit_load_skb_data_hlen(struct bpf_jit *jit)
                   offsetof(struct sk_buff, data_len));
        /* stg %w1,ST_OFF_HLEN(%r0,%r15) */
        EMIT6_DISP_LH(0xe3000000, 0x0024, REG_W1, REG_0, REG_15, STK_OFF_HLEN);
-       /* lg %skb_data,data_off(%b1) */
-       EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
-                     BPF_REG_1, offsetof(struct sk_buff, data));
+       if (!(jit->seen & SEEN_REG_AX))
+               /* lg %skb_data,data_off(%b1) */
+               EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
+                             BPF_REG_1, offsetof(struct sk_buff, data));
 }
 
 /*
@@ -487,6 +491,8 @@ static noinline int bpf_jit_insn(struct bpf_jit *jit, struct bpf_prog *fp, int i
        s32 imm = insn->imm;
        s16 off = insn->off;
 
+       if (dst_reg == BPF_REG_AX || src_reg == BPF_REG_AX)
+               jit->seen |= SEEN_REG_AX;
        switch (insn->code) {
        /*
         * BPF_MOV
@@ -1188,7 +1194,7 @@ call_fn:
                /*
                 * Implicit input:
                 *  BPF_REG_6    (R7) : skb pointer
-                *  REG_SKB_DATA (R12): skb data pointer
+                *  REG_SKB_DATA (R12): skb data pointer (if no BPF_REG_AX)
                 *
                 * Calculated input:
                 *  BPF_REG_2    (R3) : offset of byte(s) to fetch in skb
@@ -1209,6 +1215,11 @@ call_fn:
                        /* agfr %b2,%src (%src is s32 here) */
                        EMIT4(0xb9180000, BPF_REG_2, src_reg);
 
+               /* Reload REG_SKB_DATA if BPF_REG_AX is used */
+               if (jit->seen & SEEN_REG_AX)
+                       /* lg %skb_data,data_off(%b6) */
+                       EMIT6_DISP_LH(0xe3000000, 0x0004, REG_SKB_DATA, REG_0,
+                                     BPF_REG_6, offsetof(struct sk_buff, data));
                /* basr %b5,%w1 (%b5 is call saved) */
                EMIT2(0x0d00, BPF_REG_5, REG_W1);
 
@@ -1264,36 +1275,60 @@ void bpf_jit_compile(struct bpf_prog *fp)
  */
 struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
 {
+       struct bpf_prog *tmp, *orig_fp = fp;
        struct bpf_binary_header *header;
+       bool tmp_blinded = false;
        struct bpf_jit jit;
        int pass;
 
        if (!bpf_jit_enable)
-               return fp;
+               return orig_fp;
+
+       tmp = bpf_jit_blind_constants(fp);
+       /*
+        * If blinding was requested and we failed during blinding,
+        * we must fall back to the interpreter.
+        */
+       if (IS_ERR(tmp))
+               return orig_fp;
+       if (tmp != fp) {
+               tmp_blinded = true;
+               fp = tmp;
+       }
 
        memset(&jit, 0, sizeof(jit));
        jit.addrs = kcalloc(fp->len + 1, sizeof(*jit.addrs), GFP_KERNEL);
-       if (jit.addrs == NULL)
-               return fp;
+       if (jit.addrs == NULL) {
+               fp = orig_fp;
+               goto out;
+       }
        /*
         * Three initial passes:
         *   - 1/2: Determine clobbered registers
         *   - 3:   Calculate program size and addrs arrray
         */
        for (pass = 1; pass <= 3; pass++) {
-               if (bpf_jit_prog(&jit, fp))
+               if (bpf_jit_prog(&jit, fp)) {
+                       fp = orig_fp;
                        goto free_addrs;
+               }
        }
        /*
         * Final pass: Allocate and generate program
         */
-       if (jit.size >= BPF_SIZE_MAX)
+       if (jit.size >= BPF_SIZE_MAX) {
+               fp = orig_fp;
                goto free_addrs;
+       }
        header = bpf_jit_binary_alloc(jit.size, &jit.prg_buf, 2, jit_fill_hole);
-       if (!header)
+       if (!header) {
+               fp = orig_fp;
                goto free_addrs;
-       if (bpf_jit_prog(&jit, fp))
+       }
+       if (bpf_jit_prog(&jit, fp)) {
+               fp = orig_fp;
                goto free_addrs;
+       }
        if (bpf_jit_enable > 1) {
                bpf_jit_dump(fp->len, jit.size, pass, jit.prg_buf);
                if (jit.prg_buf)
@@ -1306,6 +1341,10 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *fp)
        }
 free_addrs:
        kfree(jit.addrs);
+out:
+       if (tmp_blinded)
+               bpf_jit_prog_release_other(fp, fp == orig_fp ?
+                                          tmp : orig_fp);
        return fp;
 }