arm64: errata: Add workaround for Arm errata 3194386 and 3312417

Cortex-X4 and Neoverse-V3 suffer from errata whereby an MSR to the SSBS
special-purpose register does not affect subsequent speculative
instructions, permitting speculative store bypassing for a window of
time. This is described in their Software Developer Errata Notice (SDEN)
documents:

* Cortex-X4 SDEN v8.0, erratum 3194386:
  https://developer.arm.com/documentation/SDEN-2432808/0800/

* Neoverse-V3 SDEN v6.0, erratum 3312417:
  https://developer.arm.com/documentation/SDEN-2891958/0600/

To workaround these errata, it is necessary to place a speculation
barrier (SB) after MSR to the SSBS special-purpose register. This patch
adds the requisite SB after writes to SSBS within the kernel, and hides
the presence of SSBS from EL0 such that userspace software which cares
about SSBS will manipulate this via prctl(PR_GET_SPECULATION_CTRL, ...).

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: James Morse <james.morse@arm.com>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20240508081400.235362-5-mark.rutland@arm.com
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index d33e27c5ce61087df00bab09264b49be2c4194b8..eb8af8032c3156bae5eec2036c127a882d62baac 100644 (file)
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -140,6 +140,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-X2       | #2224489        | ARM64_ERRATUM_2224489       |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Cortex-X4       | #3194386        | ARM64_ERRATUM_3194386       |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1188873,1418040| ARM64_ERRATUM_1418040       |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-N1     | #1349291        | N/A                         |
@@ -156,6 +158,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Neoverse-V1     | #1619801        | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
+| ARM            | Neoverse-V3     | #3312417        | ARM64_ERRATUM_3312417       |
++----------------+-----------------+-----------------+-----------------------------+
 | ARM            | MMU-500         | #841119,826419  | N/A                         |
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | MMU-600         | #1076982,1209401| N/A                         |

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 7b11c98b3e84bf76bcd6a33b34af257a9b7f48d7..89410e16d390bca34923b74e812fca1261f78bec 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -1064,6 +1064,48 @@ config ARM64_ERRATUM_3117295
 
          If unsure, say Y.
 
+config ARM64_WORKAROUND_SPECULATIVE_SSBS
+       bool
+
+config ARM64_ERRATUM_3194386
+       bool "Cortex-X4: 3194386: workaround for MSR SSBS not self-synchronizing"
+       select ARM64_WORKAROUND_SPECULATIVE_SSBS
+       default y
+       help
+         This option adds the workaround for ARM Cortex-X4 erratum 3194386.
+
+         On affected cores "MSR SSBS, #0" instructions may not affect
+         subsequent speculative instructions, which may permit unexepected
+         speculative store bypassing.
+
+         Work around this problem by placing a speculation barrier after
+         kernel changes to SSBS. The presence of the SSBS special-purpose
+         register is hidden from hwcaps and EL0 reads of ID_AA64PFR1_EL1, such
+         that userspace will use the PR_SPEC_STORE_BYPASS prctl to change
+         SSBS.
+
+         If unsure, say Y.
+
+config ARM64_ERRATUM_3312417
+       bool "Neoverse-V3: 3312417: workaround for MSR SSBS not self-synchronizing"
+       select ARM64_WORKAROUND_SPECULATIVE_SSBS
+       default y
+       help
+         This option adds the workaround for ARM Neoverse-V3 erratum 3312417.
+
+         On affected cores "MSR SSBS, #0" instructions may not affect
+         subsequent speculative instructions, which may permit unexepected
+         speculative store bypassing.
+
+         Work around this problem by placing a speculation barrier after
+         kernel changes to SSBS. The presence of the SSBS special-purpose
+         register is hidden from hwcaps and EL0 reads of ID_AA64PFR1_EL1, such
+         that userspace will use the PR_SPEC_STORE_BYPASS prctl to change
+         SSBS.
+
+         If unsure, say Y.
+
+
 config CAVIUM_ERRATUM_22375
        bool "Cavium erratum 22375, 24313"
        default y

diff --git a/arch/arm64/include/asm/cpucaps.h b/arch/arm64/include/asm/cpucaps.h
index 270680e2b5c4acca5f4ee48020f0d260acb895e8..7529c02639332fdd32d97b836bc78486400c4394 100644 (file)
--- a/arch/arm64/include/asm/cpucaps.h
+++ b/arch/arm64/include/asm/cpucaps.h
@@ -58,6 +58,8 @@ cpucap_is_possible(const unsigned int cap)
                return IS_ENABLED(CONFIG_NVIDIA_CARMEL_CNP_ERRATUM);
        case ARM64_WORKAROUND_REPEAT_TLBI:
                return IS_ENABLED(CONFIG_ARM64_WORKAROUND_REPEAT_TLBI);
+       case ARM64_WORKAROUND_SPECULATIVE_SSBS:
+               return IS_ENABLED(CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS);
        }
 
        return true;

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index 76b8dd37092ad2a9dd6e59a92d1c1fab887589da..828be635e7e1d938b7e812f6240cd9d7a070e323 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -432,6 +432,18 @@ static const struct midr_range erratum_spec_unpriv_load_list[] = {
 };
 #endif
 
+#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS
+static const struct midr_range erratum_spec_ssbs_list[] = {
+#ifdef CONFIG_ARM64_ERRATUM_3194386
+       MIDR_ALL_VERSIONS(MIDR_CORTEX_X4),
+#endif
+#ifdef CONFIG_ARM64_ERRATUM_3312417
+       MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3),
+#endif
+       {}
+};
+#endif
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
        {
@@ -729,6 +741,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                MIDR_FIXED(MIDR_CPU_VAR_REV(1,1), BIT(25)),
        },
 #endif
+#ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS
+       {
+               .desc = "ARM errata 3194386, 3312417",
+               .capability = ARM64_WORKAROUND_SPECULATIVE_SSBS,
+               ERRATA_MIDR_RANGE_LIST(erratum_spec_ssbs_list),
+       },
+#endif
 #ifdef CONFIG_ARM64_WORKAROUND_SPECULATIVE_UNPRIV_LOAD
        {
                .desc = "ARM errata 2966298, 3117295",

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index 56583677c1f294d66bd5ac50741089243bf5e0cd..48e7029f10548d18691bca79f6cf1a47e8941cb7 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -2307,6 +2307,14 @@ static void user_feature_fixup(void)
                if (regp)
                        regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK;
        }
+
+       if (cpus_have_cap(ARM64_WORKAROUND_SPECULATIVE_SSBS)) {
+               struct arm64_ftr_reg *regp;
+
+               regp = get_arm64_ftr_reg(SYS_ID_AA64PFR1_EL1);
+               if (regp)
+                       regp->user_mask &= ~ID_AA64PFR1_EL1_SSBS_MASK;
+       }
 }
 
 static void elf_hwcap_fixup(void)

diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 6268a13a1d5892866949b9ff8f59e64343df472b..baca47bd443c86e60fddd4e593dfc8faada1fc7a 100644 (file)
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -558,6 +558,18 @@ static enum mitigation_state spectre_v4_enable_hw_mitigation(void)
 
        /* SCTLR_EL1.DSSBS was initialised to 0 during boot */
        set_pstate_ssbs(0);
+
+       /*
+        * SSBS is self-synchronizing and is intended to affect subsequent
+        * speculative instructions, but some CPUs can speculate with a stale
+        * value of SSBS.
+        *
+        * Mitigate this with an unconditional speculation barrier, as CPUs
+        * could mis-speculate branches and bypass a conditional barrier.
+        */
+       if (IS_ENABLED(CONFIG_ARM64_WORKAROUND_SPECULATIVE_SSBS))
+               spec_bar();
+
        return SPECTRE_MITIGATED;
 }
 

diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 62b2838a231adae55ca8493c2409ae7f4b70d82e..ac3429d892b9a7c4eba7b1026b45753eca5bb7fc 100644 (file)
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -102,4 +102,5 @@ WORKAROUND_NVIDIA_CARMEL_CNP
 WORKAROUND_QCOM_FALKOR_E1003
 WORKAROUND_REPEAT_TLBI
 WORKAROUND_SPECULATIVE_AT
+WORKAROUND_SPECULATIVE_SSBS
 WORKAROUND_SPECULATIVE_UNPRIV_LOAD