riscv: Implement Shadow Call Stack

Implement CONFIG_SHADOW_CALL_STACK for RISC-V. When enabled, the
compiler injects instructions to all non-leaf C functions to
store the return address to the shadow stack and unconditionally
load it again before returning, which makes it harder to corrupt
the return address through a stack overflow, for example.

The active shadow call stack pointer is stored in the gp
register, which makes SCS incompatible with gp relaxation. Use
--no-relax-gp to ensure gp relaxation is disabled and disable
global pointer loading.  Add SCS pointers to struct thread_info,
implement SCS initialization, and task switching

Signed-off-by: Sami Tolvanen <samitolvanen@google.com>
Tested-by: Nathan Chancellor <nathan@kernel.org>
Link: https://lore.kernel.org/r/20230927224757.1154247-12-samitolvanen@google.com
Signed-off-by: Palmer Dabbelt <palmer@rivosinc.com>

diff --git a/arch/riscv/Kconfig b/arch/riscv/Kconfig
index d607ab0f7c6dafa4f754f34a910ef66464e5a06d..2f14ccb5fb01eaba4e77f0f1290fbf22688ec1ec 100644 (file)
--- a/arch/riscv/Kconfig
+++ b/arch/riscv/Kconfig
@@ -48,6 +48,7 @@ config RISCV
        select ARCH_SUPPORTS_HUGETLBFS if MMU
        select ARCH_SUPPORTS_PAGE_TABLE_CHECK if MMU
        select ARCH_SUPPORTS_PER_VMA_LOCK if MMU
+       select ARCH_SUPPORTS_SHADOW_CALL_STACK if HAVE_SHADOW_CALL_STACK
        select ARCH_USE_MEMTEST
        select ARCH_USE_QUEUED_RWLOCKS
        select ARCH_USES_CFI_TRAPS if CFI_CLANG
@@ -174,6 +175,11 @@ config GCC_SUPPORTS_DYNAMIC_FTRACE
        def_bool CC_IS_GCC
        depends on $(cc-option,-fpatchable-function-entry=8)
 
+config HAVE_SHADOW_CALL_STACK
+       def_bool $(cc-option,-fsanitize=shadow-call-stack)
+       # https://github.com/riscv-non-isa/riscv-elf-psabi-doc/commit/a484e843e6eeb51f0cb7b8819e50da6d2444d769
+       depends on $(ld-option,--no-relax-gp)
+
 config ARCH_MMAP_RND_BITS_MIN
        default 18 if 64BIT
        default 8

diff --git a/arch/riscv/Makefile b/arch/riscv/Makefile
index 1329e060c5482dd4ac7f8c3eeff17b4d00215d33..304b9444650748271ed8d3b1bc944fa09ee01062 100644 (file)
--- a/arch/riscv/Makefile
+++ b/arch/riscv/Makefile
@@ -55,6 +55,10 @@ endif
 endif
 endif
 
+ifeq ($(CONFIG_SHADOW_CALL_STACK),y)
+       KBUILD_LDFLAGS += --no-relax-gp
+endif
+
 # ISA string setting
 riscv-march-$(CONFIG_ARCH_RV32I)       := rv32ima
 riscv-march-$(CONFIG_ARCH_RV64I)       := rv64ima

diff --git a/arch/riscv/include/asm/asm.h b/arch/riscv/include/asm/asm.h
index f34dd1a526a125f8a357bb860d0c474ff04b9415..b0487b39e6747ae384fb51f310bf784759825240 100644 (file)
--- a/arch/riscv/include/asm/asm.h
+++ b/arch/riscv/include/asm/asm.h
@@ -109,6 +109,11 @@
        REG_L \dst, 0(\dst)
 .endm
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+/* gp is used as the shadow call stack pointer instead */
+.macro load_global_pointer
+.endm
+#else
 /* load __global_pointer to gp */
 .macro load_global_pointer
 .option push
@@ -116,6 +121,7 @@
        la gp, __global_pointer$
 .option pop
 .endm
+#endif /* CONFIG_SHADOW_CALL_STACK */
 
        /* save all GPs except x1 ~ x5 */
        .macro save_from_x6_to_x31

diff --git a/arch/riscv/include/asm/scs.h b/arch/riscv/include/asm/scs.h
new file mode 100644 (file)
index 0000000..94726ea
--- /dev/null
+++ b/arch/riscv/include/asm/scs.h
@@ -0,0 +1,47 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+#ifndef _ASM_SCS_H
+#define _ASM_SCS_H
+
+#ifdef __ASSEMBLY__
+#include <asm/asm-offsets.h>
+
+#ifdef CONFIG_SHADOW_CALL_STACK
+
+/* Load init_shadow_call_stack to gp. */
+.macro scs_load_init_stack
+       la      gp, init_shadow_call_stack
+       XIP_FIXUP_OFFSET gp
+.endm
+
+/* Load task_scs_sp(current) to gp. */
+.macro scs_load_current
+       REG_L   gp, TASK_TI_SCS_SP(tp)
+.endm
+
+/* Load task_scs_sp(current) to gp, but only if tp has changed. */
+.macro scs_load_current_if_task_changed prev
+       beq     \prev, tp, _skip_scs
+       scs_load_current
+_skip_scs:
+.endm
+
+/* Save gp to task_scs_sp(current). */
+.macro scs_save_current
+       REG_S   gp, TASK_TI_SCS_SP(tp)
+.endm
+
+#else /* CONFIG_SHADOW_CALL_STACK */
+
+.macro scs_load_init_stack
+.endm
+.macro scs_load_current
+.endm
+.macro scs_load_current_if_task_changed prev
+.endm
+.macro scs_save_current
+.endm
+
+#endif /* CONFIG_SHADOW_CALL_STACK */
+#endif /* __ASSEMBLY__ */
+
+#endif /* _ASM_SCS_H */

diff --git a/arch/riscv/include/asm/thread_info.h b/arch/riscv/include/asm/thread_info.h
index d18ce0113ca1f1d74d4009fff9d2081447b2807d..574779900bfb339eeb446e49f4aae119fe382ae3 100644 (file)
--- a/arch/riscv/include/asm/thread_info.h
+++ b/arch/riscv/include/asm/thread_info.h
@@ -57,8 +57,20 @@ struct thread_info {
        long                    user_sp;        /* User stack pointer */
        int                     cpu;
        unsigned long           syscall_work;   /* SYSCALL_WORK_ flags */
+#ifdef CONFIG_SHADOW_CALL_STACK
+       void                    *scs_base;
+       void                    *scs_sp;
+#endif
 };
 
+#ifdef CONFIG_SHADOW_CALL_STACK
+#define INIT_SCS                                                       \
+       .scs_base       = init_shadow_call_stack,                       \
+       .scs_sp         = init_shadow_call_stack,
+#else
+#define INIT_SCS
+#endif
+
 /*
  * macros/functions for gaining access to the thread information structure
  *
@@ -68,6 +80,7 @@ struct thread_info {
 {                                              \
        .flags          = 0,                    \
        .preempt_count  = INIT_PREEMPT_COUNT,   \
+       INIT_SCS                                \
 }
 
 void arch_release_task_struct(struct task_struct *tsk);

diff --git a/arch/riscv/kernel/asm-offsets.c b/arch/riscv/kernel/asm-offsets.c
index 0af8860f9d68b561fba69495916d728178e5b2fe..a03129f40c464868e9352436e1273424c1da060b 100644 (file)
--- a/arch/riscv/kernel/asm-offsets.c
+++ b/arch/riscv/kernel/asm-offsets.c
@@ -39,6 +39,9 @@ void asm_offsets(void)
        OFFSET(TASK_TI_PREEMPT_COUNT, task_struct, thread_info.preempt_count);
        OFFSET(TASK_TI_KERNEL_SP, task_struct, thread_info.kernel_sp);
        OFFSET(TASK_TI_USER_SP, task_struct, thread_info.user_sp);
+#ifdef CONFIG_SHADOW_CALL_STACK
+       OFFSET(TASK_TI_SCS_SP, task_struct, thread_info.scs_sp);
+#endif
 
        OFFSET(TASK_TI_CPU_NUM, task_struct, thread_info.cpu);
        OFFSET(TASK_THREAD_F0,  task_struct, thread.fstate.f[0]);

diff --git a/arch/riscv/kernel/entry.S b/arch/riscv/kernel/entry.S
index 6215dcf2e83becb46eeb39301c15495e6e20ef3d..52793193a763a6b82b35b8d0bca33ac561bac199 100644 (file)
--- a/arch/riscv/kernel/entry.S
+++ b/arch/riscv/kernel/entry.S
@@ -9,6 +9,7 @@
 
 #include <asm/asm.h>
 #include <asm/csr.h>
+#include <asm/scs.h>
 #include <asm/unistd.h>
 #include <asm/page.h>
 #include <asm/thread_info.h>
@@ -77,6 +78,9 @@ _save_context:
        /* Load the global pointer */
        load_global_pointer
 
+       /* Load the kernel shadow call stack pointer if coming from userspace */
+       scs_load_current_if_task_changed s5
+
        move a0, sp /* pt_regs */
        la ra, ret_from_exception
 
@@ -123,6 +127,9 @@ SYM_CODE_START_NOALIGN(ret_from_exception)
        addi s0, sp, PT_SIZE_ON_STACK
        REG_S s0, TASK_TI_KERNEL_SP(tp)
 
+       /* Save the kernel shadow call stack pointer */
+       scs_save_current
+
        /*
         * Save TP into the scratch register , so we can find the kernel data
         * structures again.
@@ -275,6 +282,8 @@ SYM_FUNC_START(__switch_to)
        REG_S s9,  TASK_THREAD_S9_RA(a3)
        REG_S s10, TASK_THREAD_S10_RA(a3)
        REG_S s11, TASK_THREAD_S11_RA(a3)
+       /* Save the kernel shadow call stack pointer */
+       scs_save_current
        /* Restore context from next->thread */
        REG_L ra,  TASK_THREAD_RA_RA(a4)
        REG_L sp,  TASK_THREAD_SP_RA(a4)
@@ -292,6 +301,8 @@ SYM_FUNC_START(__switch_to)
        REG_L s11, TASK_THREAD_S11_RA(a4)
        /* The offset of thread_info in task_struct is zero. */
        move tp, a1
+       /* Switch to the next shadow call stack */
+       scs_load_current
        ret
 SYM_FUNC_END(__switch_to)
 

diff --git a/arch/riscv/kernel/head.S b/arch/riscv/kernel/head.S
index a0484d95d8fb3dd859b5de48421fe9315cca26ea..18f97ec0f7ed5e46049cdbe295239ad8b7c24b2c 100644 (file)
--- a/arch/riscv/kernel/head.S
+++ b/arch/riscv/kernel/head.S
@@ -14,6 +14,7 @@
 #include <asm/cpu_ops_sbi.h>
 #include <asm/hwcap.h>
 #include <asm/image.h>
+#include <asm/scs.h>
 #include <asm/xip_fixup.h>
 #include "efi-header.S"
 
@@ -153,6 +154,7 @@ secondary_start_sbi:
        XIP_FIXUP_OFFSET a3
        add a3, a3, a1
        REG_L sp, (a3)
+       scs_load_current
 
 .Lsecondary_start_common:
 
@@ -289,6 +291,7 @@ clear_bss_done:
        la sp, init_thread_union + THREAD_SIZE
        XIP_FIXUP_OFFSET sp
        addi sp, sp, -PT_SIZE_ON_STACK
+       scs_load_init_stack
 #ifdef CONFIG_BUILTIN_DTB
        la a0, __dtb_start
        XIP_FIXUP_OFFSET a0
@@ -307,6 +310,7 @@ clear_bss_done:
        la tp, init_task
        la sp, init_thread_union + THREAD_SIZE
        addi sp, sp, -PT_SIZE_ON_STACK
+       scs_load_current
 
 #ifdef CONFIG_KASAN
        call kasan_early_init

diff --git a/arch/riscv/kernel/vdso/Makefile b/arch/riscv/kernel/vdso/Makefile
index 6b1dba11bf6dcde2baea248ce1395f5f334ca04f..48c362c0cb3d43a2f9639d4697859ce44fe860e3 100644 (file)
--- a/arch/riscv/kernel/vdso/Makefile
+++ b/arch/riscv/kernel/vdso/Makefile
@@ -36,7 +36,7 @@ CPPFLAGS_vdso.lds += -DHAS_VGETTIMEOFDAY
 endif
 
 # Disable -pg to prevent insert call site
-CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE)
+CFLAGS_REMOVE_vgettimeofday.o = $(CC_FLAGS_FTRACE) $(CC_FLAGS_SCS)
 
 # Disable profiling and instrumentation for VDSO code
 GCOV_PROFILE := n

diff --git a/arch/riscv/purgatory/Makefile b/arch/riscv/purgatory/Makefile
index 9e6476719abbbb4e9206cd89292be7f25261a5b6..6a3c16bd5ca3bb7fbd357e48f6b19cd5e7a360b0 100644 (file)
--- a/arch/riscv/purgatory/Makefile
+++ b/arch/riscv/purgatory/Makefile
@@ -81,6 +81,10 @@ ifdef CONFIG_CFI_CLANG
 PURGATORY_CFLAGS_REMOVE                += $(CC_FLAGS_CFI)
 endif
 
+ifdef CONFIG_SHADOW_CALL_STACK
+PURGATORY_CFLAGS_REMOVE                += $(CC_FLAGS_SCS)
+endif
+
 CFLAGS_REMOVE_purgatory.o      += $(PURGATORY_CFLAGS_REMOVE)
 CFLAGS_purgatory.o             += $(PURGATORY_CFLAGS)