powerpc/64s: Zeroise gprs on interrupt routine entry on Book3S

Zeroise user state in gprs (assign to zero) to reduce the influence of user
registers on speculation within kernel syscall handlers. Clears occur
at the very beginning of the sc and scv 0 interrupt handlers, with
restores occurring following the execution of the syscall handler.

Zeroise GPRS r0, r2-r11, r14-r31, on entry into the kernel for all
other interrupt sources. The remaining gprs are overwritten by
entry macros to interrupt handlers, irrespective of whether or not a
given handler consumes these register values. If an interrupt does not
select the IMSR_R12 IOption, zeroise r12.

Prior to this commit, r14-r31 are restored on a per-interrupt basis at
exit, but now they are always restored on 64bit Book3S. Remove explicit
REST_NVGPRS invocations on 64-bit Book3S. 32-bit systems do not clear
user registers on interrupt, and continue to depend on the return value
of interrupt_exit_user_prepare to determine whether or not to restore
non-volatiles.

The mmap_bench benchmark in selftests should rapidly invoke pagefaults.
See ~0.8% performance regression with this mitigation, but this
indicates the worst-case performance due to heavier-weight interrupt
handlers. This mitigation is able to be enabled/disabled through
CONFIG_INTERRUPT_SANITIZE_REGISTERS.

Reviewed-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Rohan McLure <rmclure@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20221201071019.1953023-5-rmclure@linux.ibm.com

diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S
index 42b7c3212f296987dab81b901b3b36c27ccb6da3..429096b037d7a9899c78f7345164c77a3d2a6152 100644 (file)
--- a/arch/powerpc/kernel/exceptions-64s.S
+++ b/arch/powerpc/kernel/exceptions-64s.S
@@ -506,6 +506,7 @@ DEFINE_FIXED_SYMBOL(\name\()_common_real, text)
        std     r10,0(r1)               /* make stack chain pointer     */
        std     r0,GPR0(r1)             /* save r0 in stackframe        */
        std     r10,GPR1(r1)            /* save r1 in stackframe        */
+       SANITIZE_GPR(0)
 
        /* Mark our [H]SRRs valid for return */
        li      r10,1
@@ -548,8 +549,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
        std     r9,GPR11(r1)
        std     r10,GPR12(r1)
        std     r11,GPR13(r1)
+       .if !IMSR_R12
+       SANITIZE_GPRS(9, 12)
+       .else
+       SANITIZE_GPRS(9, 11)
+       .endif
 
        SAVE_NVGPRS(r1)
+       SANITIZE_NVGPRS()
 
        .if IDAR
        .if IISIDE
@@ -581,8 +588,8 @@ BEGIN_FTR_SECTION
 END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
        ld      r10,IAREA+EX_CTR(r13)
        std     r10,_CTR(r1)
-       std     r2,GPR2(r1)             /* save r2 in stackframe        */
-       SAVE_GPRS(3, 8, r1)             /* save r3 - r8 in stackframe   */
+       SAVE_GPRS(2, 8, r1)             /* save r2 - r8 in stackframe   */
+       SANITIZE_GPRS(2, 8)
        mflr    r9                      /* Get LR, later save to stack  */
        LOAD_PACA_TOC()                 /* get kernel TOC into r2       */
        std     r9,_LINK(r1)
@@ -700,6 +707,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
        mtlr    r9
        ld      r9,_CCR(r1)
        mtcr    r9
+       SANITIZE_RESTORE_NVGPRS()
        REST_GPRS(2, 13, r1)
        REST_GPR(0, r1)
        /* restore original r1. */
@@ -1445,7 +1453,7 @@ ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_TYPE_RADIX)
         * do_break() may have changed the NV GPRS while handling a breakpoint.
         * If so, we need to restore them with their updated values.
         */
-       REST_NVGPRS(r1)
+       HANDLER_RESTORE_NVGPRS()
        b       interrupt_return_srr
 
 
@@ -1671,7 +1679,7 @@ EXC_COMMON_BEGIN(alignment_common)
        GEN_COMMON alignment
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      alignment_exception
-       REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+       HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
        b       interrupt_return_srr
 
 
@@ -1737,7 +1745,7 @@ EXC_COMMON_BEGIN(program_check_common)
 .Ldo_program_check:
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      program_check_exception
-       REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+       HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
        b       interrupt_return_srr
 
 
@@ -2169,7 +2177,7 @@ EXC_COMMON_BEGIN(emulation_assist_common)
        GEN_COMMON emulation_assist
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      emulation_assist_interrupt
-       REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+       HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
        b       interrupt_return_hsrr
 
 
@@ -2501,7 +2509,7 @@ EXC_COMMON_BEGIN(facility_unavailable_common)
        GEN_COMMON facility_unavailable
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      facility_unavailable_exception
-       REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+       HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
        b       interrupt_return_srr
 
 
@@ -2529,7 +2537,8 @@ EXC_COMMON_BEGIN(h_facility_unavailable_common)
        GEN_COMMON h_facility_unavailable
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      facility_unavailable_exception
-       REST_NVGPRS(r1) /* XXX Shouldn't be necessary in practice */
+       /* XXX Shouldn't be necessary in practice */
+       HANDLER_RESTORE_NVGPRS()
        b       interrupt_return_hsrr
 
 
@@ -2755,7 +2764,7 @@ EXC_COMMON_BEGIN(altivec_assist_common)
        addi    r3,r1,STACK_INT_FRAME_REGS
 #ifdef CONFIG_ALTIVEC
        bl      altivec_assist_exception
-       REST_NVGPRS(r1) /* instruction emulation may change GPRs */
+       HANDLER_RESTORE_NVGPRS() /* instruction emulation may change GPRs */
 #else
        bl      unknown_exception
 #endif

diff --git a/arch/powerpc/kernel/interrupt_64.S b/arch/powerpc/kernel/interrupt_64.S
index dd04b0ba3959ac41927743f2e3b000f30b30a4b1..fccc34489add8456bb060b8c0a476acfc3428a88 100644 (file)
--- a/arch/powerpc/kernel/interrupt_64.S
+++ b/arch/powerpc/kernel/interrupt_64.S
@@ -96,6 +96,11 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
         * but this is the best we can do.
         */
 
+       /*
+        * Zero user registers to prevent influencing speculative execution
+        * state of kernel code.
+        */
+       SANITIZE_SYSCALL_GPRS()
        bl      system_call_exception
 
 .Lsyscall_vectored_\name\()_exit:
@@ -124,6 +129,7 @@ BEGIN_FTR_SECTION
        HMT_MEDIUM_LOW
 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 
+       SANITIZE_RESTORE_NVGPRS()
        cmpdi   r3,0
        bne     .Lsyscall_vectored_\name\()_restore_regs
 
@@ -159,7 +165,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
        ld      r4,_LINK(r1)
        ld      r5,_XER(r1)
 
-       REST_NVGPRS(r1)
+       HANDLER_RESTORE_NVGPRS()
        REST_GPR(0, r1)
        mtcr    r2
        mtctr   r3
@@ -275,6 +281,11 @@ END_BTB_FLUSH_SECTION
        wrteei  1
 #endif
 
+       /*
+        * Zero user registers to prevent influencing speculative execution
+        * state of kernel code.
+        */
+       SANITIZE_SYSCALL_GPRS()
        bl      system_call_exception
 
 .Lsyscall_exit:
@@ -315,6 +326,7 @@ BEGIN_FTR_SECTION
        stdcx.  r0,0,r1                 /* to clear the reservation */
 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
 
+       SANITIZE_RESTORE_NVGPRS()
        cmpdi   r3,0
        bne     .Lsyscall_restore_regs
        /* Zero volatile regs that may contain sensitive kernel data */
@@ -342,7 +354,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
 .Lsyscall_restore_regs:
        ld      r3,_CTR(r1)
        ld      r4,_XER(r1)
-       REST_NVGPRS(r1)
+       HANDLER_RESTORE_NVGPRS()
        mtctr   r3
        mtspr   SPRN_XER,r4
        REST_GPR(0, r1)