arm64: pac: Optimize kernel entry/exit key installation code paths

The kernel does not use any keys besides IA so we don't need to
install IB/DA/DB/GA on kernel exit if we arrange to install them
on task switch instead, which we can expect to happen an order of
magnitude less often.

Furthermore we can avoid installing the user IA in the case where the
user task has IA disabled and just leave the kernel IA installed. This
also lets us avoid needing to install IA on kernel entry.

On an Apple M1 under a hypervisor, the overhead of kernel entry/exit
has been measured to be reduced by 15.6ns in the case where IA is
enabled, and 31.9ns in the case where IA is disabled.

Signed-off-by: Peter Collingbourne <pcc@google.com>
Link: https://linux-review.googlesource.com/id/Ieddf6b580d23c9e0bed45a822dabe72d2ffc9a8e
Link: https://lore.kernel.org/r/2d653d055f38f779937f2b92f8ddd5cf9e4af4f4.1616123271.git.pcc@google.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/include/asm/asm_pointer_auth.h b/arch/arm64/include/asm/asm_pointer_auth.h
index 52dead2a8640d2ef4acf30a0059e2d2c3413e906..8ca2dc0661ee298a503d6f794ec8bfff95932da2 100644 (file)
--- a/arch/arm64/include/asm/asm_pointer_auth.h
+++ b/arch/arm64/include/asm/asm_pointer_auth.h
@@ -13,30 +13,12 @@
  * so use the base value of ldp as thread.keys_user and offset as
  * thread.keys_user.ap*.
  */
-       .macro ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
+       .macro __ptrauth_keys_install_user tsk, tmp1, tmp2, tmp3
        mov     \tmp1, #THREAD_KEYS_USER
        add     \tmp1, \tsk, \tmp1
-alternative_if_not ARM64_HAS_ADDRESS_AUTH
-       b       .Laddr_auth_skip_\@
-alternative_else_nop_endif
        ldp     \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIA]
        msr_s   SYS_APIAKEYLO_EL1, \tmp2
        msr_s   SYS_APIAKEYHI_EL1, \tmp3
-       ldp     \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APIB]
-       msr_s   SYS_APIBKEYLO_EL1, \tmp2
-       msr_s   SYS_APIBKEYHI_EL1, \tmp3
-       ldp     \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDA]
-       msr_s   SYS_APDAKEYLO_EL1, \tmp2
-       msr_s   SYS_APDAKEYHI_EL1, \tmp3
-       ldp     \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APDB]
-       msr_s   SYS_APDBKEYLO_EL1, \tmp2
-       msr_s   SYS_APDBKEYHI_EL1, \tmp3
-.Laddr_auth_skip_\@:
-alternative_if ARM64_HAS_GENERIC_AUTH
-       ldp     \tmp2, \tmp3, [\tmp1, #PTRAUTH_USER_KEY_APGA]
-       msr_s   SYS_APGAKEYLO_EL1, \tmp2
-       msr_s   SYS_APGAKEYHI_EL1, \tmp3
-alternative_else_nop_endif
        .endm
 
        .macro __ptrauth_keys_install_kernel_nosync tsk, tmp1, tmp2, tmp3

diff --git a/arch/arm64/include/asm/pointer_auth.h b/arch/arm64/include/asm/pointer_auth.h
index cefe7e9b6fa9c83304c26cedd87b1248e6c61358..d50416be99be0bf0d903f0a985b476546bc83d63 100644 (file)
--- a/arch/arm64/include/asm/pointer_auth.h
+++ b/arch/arm64/include/asm/pointer_auth.h
@@ -35,6 +35,25 @@ struct ptrauth_keys_kernel {
        struct ptrauth_key apia;
 };
 
+#define __ptrauth_key_install_nosync(k, v)                     \
+do {                                                           \
+       struct ptrauth_key __pki_v = (v);                       \
+       write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1);     \
+       write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1);     \
+} while (0)
+
+static inline void ptrauth_keys_install_user(struct ptrauth_keys_user *keys)
+{
+       if (system_supports_address_auth()) {
+               __ptrauth_key_install_nosync(APIB, keys->apib);
+               __ptrauth_key_install_nosync(APDA, keys->apda);
+               __ptrauth_key_install_nosync(APDB, keys->apdb);
+       }
+
+       if (system_supports_generic_auth())
+               __ptrauth_key_install_nosync(APGA, keys->apga);
+}
+
 static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys)
 {
        if (system_supports_address_auth()) {
@@ -46,14 +65,9 @@ static inline void ptrauth_keys_init_user(struct ptrauth_keys_user *keys)
 
        if (system_supports_generic_auth())
                get_random_bytes(&keys->apga, sizeof(keys->apga));
-}
 
-#define __ptrauth_key_install_nosync(k, v)                     \
-do {                                                           \
-       struct ptrauth_key __pki_v = (v);                       \
-       write_sysreg_s(__pki_v.lo, SYS_ ## k ## KEYLO_EL1);     \
-       write_sysreg_s(__pki_v.hi, SYS_ ## k ## KEYHI_EL1);     \
-} while (0)
+       ptrauth_keys_install_user(keys);
+}
 
 static __always_inline void ptrauth_keys_init_kernel(struct ptrauth_keys_kernel *keys)
 {
@@ -90,6 +104,9 @@ static __always_inline void ptrauth_enable(void)
        isb();
 }
 
+#define ptrauth_suspend_exit()                                                 \
+       ptrauth_keys_install_user(&current->thread.keys_user)
+
 #define ptrauth_thread_init_user()                                             \
        do {                                                                   \
                ptrauth_keys_init_user(&current->thread.keys_user);            \
@@ -101,6 +118,9 @@ static __always_inline void ptrauth_enable(void)
                                           SCTLR_ELx_ENDA | SCTLR_ELx_ENDB);   \
        } while (0)
 
+#define ptrauth_thread_switch_user(tsk)                                        \
+       ptrauth_keys_install_user(&(tsk)->thread.keys_user)
+
 #define ptrauth_thread_init_kernel(tsk)                                        \
        ptrauth_keys_init_kernel(&(tsk)->thread.keys_kernel)
 #define ptrauth_thread_switch_kernel(tsk)                              \
@@ -112,8 +132,10 @@ static __always_inline void ptrauth_enable(void)
 #define ptrauth_set_enabled_keys(tsk, keys, enabled)   (-EINVAL)
 #define ptrauth_get_enabled_keys(tsk)  (-EINVAL)
 #define ptrauth_strip_insn_pac(lr)     (lr)
+#define ptrauth_suspend_exit()
 #define ptrauth_thread_init_user()
 #define ptrauth_thread_init_kernel(tsk)
+#define ptrauth_thread_switch_user(tsk)
 #define ptrauth_thread_switch_kernel(tsk)
 #endif /* CONFIG_ARM64_PTR_AUTH */
 

diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
index 8b4fbad427b36f3ac881c7eec2beb4b4f258e0cf..a02573c32f7941ea7286a5bf802ed4cf18d07dea 100644 (file)
--- a/arch/arm64/kernel/asm-offsets.c
+++ b/arch/arm64/kernel/asm-offsets.c
@@ -148,10 +148,6 @@ int main(void)
 #endif
 #ifdef CONFIG_ARM64_PTR_AUTH
   DEFINE(PTRAUTH_USER_KEY_APIA,                offsetof(struct ptrauth_keys_user, apia));
-  DEFINE(PTRAUTH_USER_KEY_APIB,                offsetof(struct ptrauth_keys_user, apib));
-  DEFINE(PTRAUTH_USER_KEY_APDA,                offsetof(struct ptrauth_keys_user, apda));
-  DEFINE(PTRAUTH_USER_KEY_APDB,                offsetof(struct ptrauth_keys_user, apdb));
-  DEFINE(PTRAUTH_USER_KEY_APGA,                offsetof(struct ptrauth_keys_user, apga));
   DEFINE(PTRAUTH_KERNEL_KEY_APIA,      offsetof(struct ptrauth_keys_kernel, apia));
   BLANK();
 #endif

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 36ae1cdfaeb69d5895fa59d955a5a4cce96c90d8..2e46c75a5b4873bb2bbd08af0e0ec2894db35186 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -247,21 +247,26 @@ alternative_else_nop_endif
        check_mte_async_tcf x19, x22
        apply_ssbd 1, x22, x23
 
-       ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23
-
 #ifdef CONFIG_ARM64_PTR_AUTH
 alternative_if ARM64_HAS_ADDRESS_AUTH
        /*
         * Enable IA for in-kernel PAC if the task had it disabled. Although
         * this could be implemented with an unconditional MRS which would avoid
         * a load, this was measured to be slower on Cortex-A75 and Cortex-A76.
+        *
+        * Install the kernel IA key only if IA was enabled in the task. If IA
+        * was disabled on kernel exit then we would have left the kernel IA
+        * installed so there is no need to install it again.
         */
        ldr     x0, [tsk, THREAD_SCTLR_USER]
-       tbnz    x0, SCTLR_ELx_ENIA_SHIFT, 1f
+       tbz     x0, SCTLR_ELx_ENIA_SHIFT, 1f
+       __ptrauth_keys_install_kernel_nosync tsk, x20, x22, x23
+       b       2f
+1:
        mrs     x0, sctlr_el1
        orr     x0, x0, SCTLR_ELx_ENIA
        msr     sctlr_el1, x0
-1:
+2:
        isb
 alternative_else_nop_endif
 #endif
@@ -368,24 +373,24 @@ alternative_else_nop_endif
 3:
        scs_save tsk, x0
 
-       /*
-        * No kernel C function calls after this as user keys are set and IA may
-        * be disabled.
-        */
-       ptrauth_keys_install_user tsk, x0, x1, x2
-
 #ifdef CONFIG_ARM64_PTR_AUTH
 alternative_if ARM64_HAS_ADDRESS_AUTH
        /*
-        * IA was enabled for in-kernel PAC. Disable it now if needed.
-        * All other per-task SCTLR bits were updated on task switch.
+        * IA was enabled for in-kernel PAC. Disable it now if needed, or
+        * alternatively install the user's IA. All other per-task keys and
+        * SCTLR bits were updated on task switch.
+        *
+        * No kernel C function calls after this.
         */
        ldr     x0, [tsk, THREAD_SCTLR_USER]
-       tbnz    x0, SCTLR_ELx_ENIA_SHIFT, 1f
+       tbz     x0, SCTLR_ELx_ENIA_SHIFT, 1f
+       __ptrauth_keys_install_user tsk, x0, x1, x2
+       b       2f
+1:
        mrs     x0, sctlr_el1
        bic     x0, x0, SCTLR_ELx_ENIA
        msr     sctlr_el1, x0
-1:
+2:
 alternative_else_nop_endif
 #endif
 

diff --git a/arch/arm64/kernel/pointer_auth.c b/arch/arm64/kernel/pointer_auth.c
index f03e5bfe449019103bb874fcf22dabce12ea374a..60901ab0a7fe54f74dbe4eca272ef189390c9245 100644 (file)
--- a/arch/arm64/kernel/pointer_auth.c
+++ b/arch/arm64/kernel/pointer_auth.c
@@ -43,6 +43,7 @@ int ptrauth_prctl_reset_keys(struct task_struct *tsk, unsigned long arg)
                get_random_bytes(&keys->apdb, sizeof(keys->apdb));
        if (arg & PR_PAC_APGAKEY)
                get_random_bytes(&keys->apga, sizeof(keys->apga));
+       ptrauth_keys_install_user(keys);
 
        return 0;
 }

diff --git a/arch/arm64/kernel/process.c b/arch/arm64/kernel/process.c
index d583cbae32f6aaa2a5c877bc8306e7118c8b8a61..449b94584f9a2f54e5bb5819ac40c12edb8e38cb 100644 (file)
--- a/arch/arm64/kernel/process.c
+++ b/arch/arm64/kernel/process.c
@@ -568,6 +568,7 @@ __notrace_funcgraph struct task_struct *__switch_to(struct task_struct *prev,
        entry_task_switch(next);
        ssbs_thread_switch(next);
        erratum_1418040_thread_switch(prev, next);
+       ptrauth_thread_switch_user(next);
 
        /*
         * Complete any pending TLB or cache maintenance on this CPU in case

diff --git a/arch/arm64/kernel/suspend.c b/arch/arm64/kernel/suspend.c
index 6fdc8292b4f58077149a54285af57e4ea9fbb871..e3f72df9509d72e18b192aa106dec182ace45293 100644 (file)
--- a/arch/arm64/kernel/suspend.c
+++ b/arch/arm64/kernel/suspend.c
@@ -74,8 +74,9 @@ void notrace __cpu_suspend_exit(void)
         */
        spectre_v4_enable_mitigation(NULL);
 
-       /* Restore additional MTE-specific configuration */
+       /* Restore additional feature-specific configuration */
        mte_suspend_exit();
+       ptrauth_suspend_exit();
 }
 
 /*