s390: extend expoline to BC instructions

The BPF JIT uses a 'b <disp>(%r<x>)' instruction in the definition
of the sk_load_word and sk_load_half functions.

Add support for branch-on-condition instructions contained in the
thunk code of an expoline.

Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

diff --git a/arch/s390/include/asm/nospec-insn.h b/arch/s390/include/asm/nospec-insn.h
index 7d7640e1cf90ca15c2da067b0d122737a8dc85d0..a01f81186e86aceade03e7df8f8872d307a0d516 100644 (file)
--- a/arch/s390/include/asm/nospec-insn.h
+++ b/arch/s390/include/asm/nospec-insn.h
@@ -35,10 +35,18 @@ _LC_BR_R1 = __LC_BR_R1
        __THUNK_PROLOG_NAME __s390x_indirect_jump_r\r2\()use_r\r1
        .endm
 
+       .macro __THUNK_PROLOG_BC d0,r1,r2
+       __THUNK_PROLOG_NAME __s390x_indirect_branch_\d0\()_\r2\()use_\r1
+       .endm
+
        .macro __THUNK_BR r1,r2
        jg      __s390x_indirect_jump_r\r2\()use_r\r1
        .endm
 
+       .macro __THUNK_BC d0,r1,r2
+       jg      __s390x_indirect_branch_\d0\()_\r2\()use_\r1
+       .endm
+
        .macro __THUNK_BRASL r1,r2,r3
        brasl   \r1,__s390x_indirect_jump_r\r3\()use_r\r2
        .endm
@@ -81,6 +89,23 @@ _LC_BR_R1 = __LC_BR_R1
        .endif
        .endm
 
+       .macro  __DECODE_DRR expand,disp,reg,ruse
+       .set __decode_fail,1
+       .irp r1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+       .ifc \reg,%r\r1
+       .irp r2,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
+       .ifc \ruse,%r\r2
+       \expand \disp,\r1,\r2
+       .set __decode_fail,0
+       .endif
+       .endr
+       .endif
+       .endr
+       .if __decode_fail == 1
+       .error "__DECODE_DRR failed"
+       .endif
+       .endm
+
        .macro __THUNK_EX_BR reg,ruse
        # Be very careful when adding instructions to this macro!
        # The ALTERNATIVE replacement code has a .+10 which targets
@@ -101,17 +126,42 @@ _LC_BR_R1 = __LC_BR_R1
 555:   br      \reg
        .endm
 
+       .macro __THUNK_EX_BC disp,reg,ruse
+#ifdef CONFIG_HAVE_MARCH_Z10_FEATURES
+       exrl    0,556f
+       j       .
+#else
+       larl    \ruse,556f
+       ex      0,0(\ruse)
+       j       .
+#endif
+556:   b       \disp(\reg)
+       .endm
+
        .macro GEN_BR_THUNK reg,ruse=%r1
        __DECODE_RR __THUNK_PROLOG_BR,\reg,\ruse
        __THUNK_EX_BR \reg,\ruse
        __THUNK_EPILOG
        .endm
 
+       .macro GEN_B_THUNK disp,reg,ruse=%r1
+       __DECODE_DRR __THUNK_PROLOG_BC,\disp,\reg,\ruse
+       __THUNK_EX_BC \disp,\reg,\ruse
+       __THUNK_EPILOG
+       .endm
+
        .macro BR_EX reg,ruse=%r1
 557:   __DECODE_RR __THUNK_BR,\reg,\ruse
        .pushsection .s390_indirect_branches,"a",@progbits
        .long   557b-.
        .popsection
+       .endm
+
+        .macro B_EX disp,reg,ruse=%r1
+558:   __DECODE_DRR __THUNK_BC,\disp,\reg,\ruse
+       .pushsection .s390_indirect_branches,"a",@progbits
+       .long   558b-.
+       .popsection
        .endm
 
        .macro BASR_EX rsave,rtarget,ruse=%r1
@@ -123,10 +173,17 @@ _LC_BR_R1 = __LC_BR_R1
 
 #else
        .macro GEN_BR_THUNK reg,ruse=%r1
+       .endm
+
+       .macro GEN_B_THUNK disp,reg,ruse=%r1
        .endm
 
         .macro BR_EX reg,ruse=%r1
        br      \reg
+       .endm
+
+        .macro B_EX disp,reg,ruse=%r1
+       b       \disp(\reg)
        .endm
 
        .macro BASR_EX rsave,rtarget,ruse=%r1

diff --git a/arch/s390/kernel/nospec-branch.c b/arch/s390/kernel/nospec-branch.c
index 834cf29f25999298aa9462e2382971090356fdae..8ad6a7128b3a5eba73345e2365613284466e7489 100644 (file)
--- a/arch/s390/kernel/nospec-branch.c
+++ b/arch/s390/kernel/nospec-branch.c
@@ -93,7 +93,6 @@ static void __init_or_module __nospec_revert(s32 *start, s32 *end)
        s32 *epo;
 
        /* Second part of the instruction replace is always a nop */
-       memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x00, 0x00 }, 4);
        for (epo = start; epo < end; epo++) {
                instr = (u8 *) epo + *epo;
                if (instr[0] == 0xc0 && (instr[1] & 0x0f) == 0x04)
@@ -114,18 +113,34 @@ static void __init_or_module __nospec_revert(s32 *start, s32 *end)
                        br = thunk + (*(int *)(thunk + 2)) * 2;
                else
                        continue;
-               if (br[0] != 0x07 || (br[1] & 0xf0) != 0xf0)
+               /* Check for unconditional branch 0x07f? or 0x47f???? */
+               if ((br[0] & 0xbf) != 0x07 || (br[1] & 0xf0) != 0xf0)
                        continue;
+
+               memcpy(insnbuf + 2, (char[]) { 0x47, 0x00, 0x07, 0x00 }, 4);
                switch (type) {
                case BRCL_EXPOLINE:
-                       /* brcl to thunk, replace with br + nop */
                        insnbuf[0] = br[0];
                        insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
+                       if (br[0] == 0x47) {
+                               /* brcl to b, replace with bc + nopr */
+                               insnbuf[2] = br[2];
+                               insnbuf[3] = br[3];
+                       } else {
+                               /* brcl to br, replace with bcr + nop */
+                       }
                        break;
                case BRASL_EXPOLINE:
-                       /* brasl to thunk, replace with basr + nop */
-                       insnbuf[0] = 0x0d;
                        insnbuf[1] = (instr[1] & 0xf0) | (br[1] & 0x0f);
+                       if (br[0] == 0x47) {
+                               /* brasl to b, replace with bas + nopr */
+                               insnbuf[0] = 0x4d;
+                               insnbuf[2] = br[2];
+                               insnbuf[3] = br[3];
+                       } else {
+                               /* brasl to br, replace with basr + nop */
+                               insnbuf[0] = 0x0d;
+                       }
                        break;
                }