arm64: enable ptrauth earlier

When the kernel is compiled with pointer auth instructions, the boot CPU
needs to start using address auth very early, so change the cpucap to
account for this.

Pointer auth must be enabled before we call C functions, because it is
not possible to enter a function with pointer auth disabled and exit it
with pointer auth enabled. Note, mismatches between architected and
IMPDEF algorithms will still be caught by the cpufeature framework (the
separate *_ARCH and *_IMP_DEF cpucaps).

Note the change in behavior: if the boot CPU has address auth and a
late CPU does not, then the late CPU is parked by the cpufeature
framework. This is possible as kernel will only have NOP space intructions
for PAC so such mismatched late cpu will silently ignore those
instructions in C functions. Also, if the boot CPU does not have address
auth and the late CPU has then the late cpu will still boot but with
ptrauth feature disabled.

Leave generic authentication as a "system scope" cpucap for now, since
initially the kernel will only use address authentication.

Reviewed-by: Kees Cook <keescook@chromium.org>
Reviewed-by: Suzuki K Poulose <suzuki.poulose@arm.com>
Reviewed-by: Vincenzo Frascino <Vincenzo.Frascino@arm.com>
Signed-off-by: Kristina Martsenko <kristina.martsenko@arm.com>
[Amit: Re-worked ptrauth setup logic, comments]
Signed-off-by: Amit Daniel Kachhap <amit.kachhap@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 0b30e884e0889ae0203258360e76d849dc2f133e..87e2cbb76930554fe06c900ab7185f09e1810cb0 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1515,6 +1515,12 @@ config ARM64_PTR_AUTH
          be enabled. However, KVM guest also require VHE mode and hence
          CONFIG_ARM64_VHE=y option to use this feature.
 
+         If the feature is present on the boot CPU but not on a late CPU, then
+         the late CPU will be parked. Also, if the boot CPU does not have
+         address auth and the late CPU has then the late CPU will still boot
+         but with the feature disabled. On such a system, this option should
+         not be selected.
+
 endmenu
 
 menu "ARMv8.5 architectural features"

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 46388e65bbcddd1410048984d0065617996756ff..2b5a088053e47e33e0ec6476df9b45bab6f7eb2e 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -291,6 +291,15 @@ extern struct arm64_ftr_reg arm64_ftr_reg_ctrel0;
 #define ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE           \
        (ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PANIC_ON_CONFLICT)
 
+/*
+ * CPU feature used early in the boot based on the boot CPU. It is safe for a
+ * late CPU to have this feature even though the boot CPU hasn't enabled it,
+ * although the feature will not be used by Linux in this case. If the boot CPU
+ * has enabled this feature already, then every late CPU must have it.
+ */
+#define ARM64_CPUCAP_BOOT_CPU_FEATURE                  \
+       (ARM64_CPUCAP_SCOPE_BOOT_CPU | ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU)
+
 struct arm64_cpu_capabilities {
        const char *desc;
        u16 capability;

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index d6033f45c3cd570f2b41629d764d94fcb9b8fc5e..f6c0cb7551075f9b7e16f007f45e866b8416af8c 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -1318,12 +1318,6 @@ static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
 #endif /* CONFIG_ARM64_RAS_EXTN */
 
 #ifdef CONFIG_ARM64_PTR_AUTH
-static void cpu_enable_address_auth(struct arm64_cpu_capabilities const *cap)
-{
-       sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ENIA | SCTLR_ELx_ENIB |
-                                      SCTLR_ELx_ENDA | SCTLR_ELx_ENDB);
-}
-
 static bool has_address_auth(const struct arm64_cpu_capabilities *entry,
                             int __unused)
 {
@@ -1627,7 +1621,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
        {
                .desc = "Address authentication (architected algorithm)",
                .capability = ARM64_HAS_ADDRESS_AUTH_ARCH,
-               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+               .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
                .sys_reg = SYS_ID_AA64ISAR1_EL1,
                .sign = FTR_UNSIGNED,
                .field_pos = ID_AA64ISAR1_APA_SHIFT,
@@ -1637,7 +1631,7 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
        {
                .desc = "Address authentication (IMP DEF algorithm)",
                .capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
-               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+               .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
                .sys_reg = SYS_ID_AA64ISAR1_EL1,
                .sign = FTR_UNSIGNED,
                .field_pos = ID_AA64ISAR1_API_SHIFT,
@@ -1646,9 +1640,8 @@ static const struct arm64_cpu_capabilities arm64_features[] = {
        },
        {
                .capability = ARM64_HAS_ADDRESS_AUTH,
-               .type = ARM64_CPUCAP_SYSTEM_FEATURE,
+               .type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
                .matches = has_address_auth,
-               .cpu_enable = cpu_enable_address_auth,
        },
        {
                .desc = "Generic authentication (architected algorithm)",

diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S
index ea0db1744c297bee635f1dd1f40a1f2e32d5981a..4cf19a26af2d8659ab2869ed1daa9ee33bf37d91 100644 (file)
--- a/arch/arm64/mm/proc.S
+++ b/arch/arm64/mm/proc.S
@@ -16,6 +16,7 @@
 #include <asm/pgtable-hwdef.h>
 #include <asm/cpufeature.h>
 #include <asm/alternative.h>
+#include <asm/smp.h>
 
 #ifdef CONFIG_ARM64_64K_PAGES
 #define TCR_TG_FLAGS   TCR_TG0_64K | TCR_TG1_64K
@@ -468,9 +469,39 @@ SYM_FUNC_START(__cpu_setup)
 1:
 #endif /* CONFIG_ARM64_HW_AFDBM */
        msr     tcr_el1, x10
+       mov     x1, x0
        /*
         * Prepare SCTLR
         */
        mov_q   x0, SCTLR_EL1_SET
+
+#ifdef CONFIG_ARM64_PTR_AUTH
+       /* No ptrauth setup for run time cpus */
+       cmp     x1, #ARM64_CPU_RUNTIME
+       b.eq    3f
+
+       /* Check if the CPU supports ptrauth */
+       mrs     x2, id_aa64isar1_el1
+       ubfx    x2, x2, #ID_AA64ISAR1_APA_SHIFT, #8
+       cbz     x2, 3f
+
+       msr_s   SYS_APIAKEYLO_EL1, xzr
+       msr_s   SYS_APIAKEYHI_EL1, xzr
+
+       /* Just enable ptrauth for primary cpu */
+       cmp     x1, #ARM64_CPU_BOOT_PRIMARY
+       b.eq    2f
+
+       /* if !system_supports_address_auth() then skip enable */
+alternative_if_not ARM64_HAS_ADDRESS_AUTH
+       b       3f
+alternative_else_nop_endif
+
+2:     /* Enable ptrauth instructions */
+       ldr     x2, =SCTLR_ELx_ENIA | SCTLR_ELx_ENIB | \
+                    SCTLR_ELx_ENDA | SCTLR_ELx_ENDB
+       orr     x0, x0, x2
+3:
+#endif
        ret                                     // return to head.S
 SYM_FUNC_END(__cpu_setup)