arm64/gcs: Context switch GCS state for EL0

There are two registers controlling the GCS state of EL0, GCSPR_EL0 which
is the current GCS pointer and GCSCRE0_EL1 which has enable bits for the
specific GCS functionality enabled for EL0. Manage these on context switch
and process lifetime events, GCS is reset on exec().  Also ensure that
any changes to the GCS memory are visible to other PEs and that changes
from other PEs are visible on this one by issuing a GCSB DSYNC when
moving to or from a thread with GCS.

Since the current GCS configuration of a thread will be visible to
userspace we store the configuration in the format used with userspace
and provide a helper which configures the system register as needed.

On systems that support GCS we always allow access to GCSPR_EL0, this
facilitates reporting of GCS faults if userspace implements disabling of
GCS on error - the GCS can still be discovered and examined even if GCS
has been disabled.

Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
Reviewed-by: Thiago Jung Bauermann <thiago.bauermann@linaro.org>
Signed-off-by: Mark Brown <broonie@kernel.org>
Link: https://lore.kernel.org/r/20241001-arm64-gcs-v13-21-222b78d87eee@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/gcs.h b/arch/arm64/include/asm/gcs.h
index 7c5e95218db6b81f7ebd91de9a6cb7e39d47903f..04594ef59dadd271f1a394ea0f85c43b67842e7b 100644 (file)
--- a/arch/arm64/include/asm/gcs.h
+++ b/arch/arm64/include/asm/gcs.h
@@ -48,4 +48,28 @@ static inline u64 gcsss2(void)
        return Xt;
 }
 
+#ifdef CONFIG_ARM64_GCS
+
+static inline bool task_gcs_el0_enabled(struct task_struct *task)
+{
+       return current->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE;
+}
+
+void gcs_set_el0_mode(struct task_struct *task);
+void gcs_free(struct task_struct *task);
+void gcs_preserve_current_state(void);
+
+#else
+
+static inline bool task_gcs_el0_enabled(struct task_struct *task)
+{
+       return false;
+}
+
+static inline void gcs_set_el0_mode(struct task_struct *task) { }
+static inline void gcs_free(struct task_struct *task) { }
+static inline void gcs_preserve_current_state(void) { }
+
+#endif
+
 #endif

diff --git a/arch/arm64/include/asm/processor.h b/arch/arm64/include/asm/processor.h
index 1438424f00643708c26b0439fd697b6d97fc3ff9..5260788247d833129e60c5cd1f533f88ad2d2dc7 100644 (file)
--- a/arch/arm64/include/asm/processor.h
+++ b/arch/arm64/include/asm/processor.h
@@ -185,6 +185,12 @@ struct thread_struct {
        u64                     svcr;
        u64                     tpidr2_el0;
        u64                     por_el0;
+#ifdef CONFIG_ARM64_GCS
+       unsigned int            gcs_el0_mode;
+       u64                     gcspr_el0;
+       u64                     gcs_base;
+       u64                     gcs_size;
+#endif
 };
 
 static inline unsigned int thread_get_vl(struct thread_struct *thread,

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index 0540653fbf382bf8a0984a7e9f97cafbedc3251b..aedcf332f42270a2a85761706c498da526f1d86c 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -49,6 +49,7 @@
 #include <asm/cacheflush.h>
 #include <asm/exec.h>
 #include <asm/fpsimd.h>
+#include <asm/gcs.h>
 #include <asm/mmu_context.h>
 #include <asm/mte.h>
 #include <asm/processor.h>
@@ -280,6 +281,25 @@ static void flush_poe(void)
        write_sysreg_s(POR_EL0_INIT, SYS_POR_EL0);
 }
 
+#ifdef CONFIG_ARM64_GCS
+
+static void flush_gcs(void)
+{
+       if (!system_supports_gcs())
+               return;
+
+       gcs_free(current);
+       current->thread.gcs_el0_mode = 0;
+       write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1);
+       write_sysreg_s(0, SYS_GCSPR_EL0);
+}
+
+#else
+
+static void flush_gcs(void) { }
+
+#endif
+
 void flush_thread(void)
 {
        fpsimd_flush_thread();
@@ -287,6 +307,7 @@ void flush_thread(void)
        flush_ptrace_hw_breakpoint(current);
        flush_tagged_addr_state();
        flush_poe();
+       flush_gcs();
 }
 
 void arch_release_task_struct(struct task_struct *tsk)
@@ -484,6 +505,46 @@ static void entry_task_switch(struct task_struct *next)
        __this_cpu_write(__entry_task, next);
 }
 
+#ifdef CONFIG_ARM64_GCS
+
+void gcs_preserve_current_state(void)
+{
+       current->thread.gcspr_el0 = read_sysreg_s(SYS_GCSPR_EL0);
+}
+
+static void gcs_thread_switch(struct task_struct *next)
+{
+       if (!system_supports_gcs())
+               return;
+
+       /* GCSPR_EL0 is always readable */
+       gcs_preserve_current_state();
+       write_sysreg_s(next->thread.gcspr_el0, SYS_GCSPR_EL0);
+
+       if (current->thread.gcs_el0_mode != next->thread.gcs_el0_mode)
+               gcs_set_el0_mode(next);
+
+       /*
+        * Ensure that GCS memory effects of the 'prev' thread are
+        * ordered before other memory accesses with release semantics
+        * (or preceded by a DMB) on the current PE. In addition, any
+        * memory accesses with acquire semantics (or succeeded by a
+        * DMB) are ordered before GCS memory effects of the 'next'
+        * thread. This will ensure that the GCS memory effects are
+        * visible to other PEs in case of migration.
+        */
+       if (task_gcs_el0_enabled(current) || task_gcs_el0_enabled(next))
+               gcsb_dsync();
+}
+
+#else
+
+static void gcs_thread_switch(struct task_struct *next)
+{
+}
+
+#endif
+
 /*
  * Handle sysreg updates for ARM erratum 1418040 which affects the 32bit view of
  * CNTVCT, various other errata which require trapping all CNTVCT{,_EL0}
@@ -580,6 +641,7 @@ struct task_struct *__switch_to(struct task_struct *prev,
        cntkctl_thread_switch(prev, next);
        ptrauth_thread_switch_user(next);
        permission_overlay_switch(next);
+       gcs_thread_switch(next);
 
        /*
         * Complete any pending TLB or cache maintenance on this CPU in case

diff --git a/arch/arm64/mm/Makefile b/arch/arm64/mm/Makefile
index 2fc8c6dd04070b61078e2e8627fd3acc606adc7e..fc92170a8f377c0a6822837f33c391b520499be4 100644 (file)
--- a/arch/arm64/mm/Makefile
+++ b/arch/arm64/mm/Makefile
@@ -11,6 +11,7 @@ obj-$(CONFIG_TRANS_TABLE)     += trans_pgd.o
 obj-$(CONFIG_TRANS_TABLE)      += trans_pgd-asm.o
 obj-$(CONFIG_DEBUG_VIRTUAL)    += physaddr.o
 obj-$(CONFIG_ARM64_MTE)                += mteswap.o
+obj-$(CONFIG_ARM64_GCS)                += gcs.o
 KASAN_SANITIZE_physaddr.o      += n
 
 obj-$(CONFIG_KASAN)            += kasan_init.o

diff --git a/arch/arm64/mm/gcs.c b/arch/arm64/mm/gcs.c
new file mode 100644 (file)
index 0000000..f8f4f98
--- /dev/null
+++ b/arch/arm64/mm/gcs.c
@@ -0,0 +1,42 @@
+// SPDX-License-Identifier: GPL-2.0-only
+
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/syscalls.h>
+#include <linux/types.h>
+
+#include <asm/cpufeature.h>
+#include <asm/page.h>
+
+/*
+ * Apply the GCS mode configured for the specified task to the
+ * hardware.
+ */
+void gcs_set_el0_mode(struct task_struct *task)
+{
+       u64 gcscre0_el1 = GCSCRE0_EL1_nTR;
+
+       if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_ENABLE)
+               gcscre0_el1 |= GCSCRE0_EL1_RVCHKEN | GCSCRE0_EL1_PCRSEL;
+
+       if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_WRITE)
+               gcscre0_el1 |= GCSCRE0_EL1_STREn;
+
+       if (task->thread.gcs_el0_mode & PR_SHADOW_STACK_PUSH)
+               gcscre0_el1 |= GCSCRE0_EL1_PUSHMEn;
+
+       write_sysreg_s(gcscre0_el1, SYS_GCSCRE0_EL1);
+}
+
+void gcs_free(struct task_struct *task)
+{
+       if (!system_supports_gcs())
+               return;
+
+       if (task->thread.gcs_base)
+               vm_munmap(task->thread.gcs_base, task->thread.gcs_size);
+
+       task->thread.gcspr_el0 = 0;
+       task->thread.gcs_base = 0;
+       task->thread.gcs_size = 0;
+}