arm64: Use the clearbhb instruction in mitigations

commit 228a26b912287934789023b4132ba76065d9491c upstream.

Future CPUs may implement a clearbhb instruction that is sufficient
to mitigate SpectreBHB. CPUs that implement this instruction, but
not CSV2.3 must be affected by Spectre-BHB.

Add support to use this instruction as the BHB mitigation on CPUs
that support it. The instruction is in the hint space, so it will
be treated by a NOP as older CPUs.

Reviewed-by: Russell King (Oracle) <rmk+kernel@armlinux.org.uk>
Reviewed-by: Catalin Marinas <catalin.marinas@arm.com>
[ modified for stable: Use a KVM vector template instead of alternatives,
  removed bitmap of mitigations ]
Signed-off-by: James Morse <james.morse@arm.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/arm64/include/asm/assembler.h b/arch/arm64/include/asm/assembler.h
index 19feb2deca480492db3c0609bc82f694190797b8..011e681a23366bd3519fd9324f5047547be97b3b 100644 (file)
--- a/arch/arm64/include/asm/assembler.h
+++ b/arch/arm64/include/asm/assembler.h
@@ -97,6 +97,13 @@
        hint    #20
        .endm
 
+/*
+ * Clear Branch History instruction
+ */
+       .macro clearbhb
+       hint    #22
+       .endm
+
 /*
  * Speculation barrier
  */

diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h
index 74e33de887783c8805badbe2f0f22f1e44782a50..423f9b40e4d951a8403263cf2b7d4d9a40452487 100644 (file)
--- a/arch/arm64/include/asm/cpufeature.h
+++ b/arch/arm64/include/asm/cpufeature.h
@@ -621,6 +621,19 @@ static inline bool supports_csv2p3(int scope)
        return csv2_val == 3;
 }
 
+static inline bool supports_clearbhb(int scope)
+{
+       u64 isar2;
+
+       if (scope == SCOPE_LOCAL_CPU)
+               isar2 = read_sysreg_s(SYS_ID_AA64ISAR2_EL1);
+       else
+               isar2 = read_sanitised_ftr_reg(SYS_ID_AA64ISAR2_EL1);
+
+       return cpuid_feature_extract_unsigned_field(isar2,
+                                                   ID_AA64ISAR2_CLEARBHB_SHIFT);
+}
+
 static inline bool system_supports_32bit_el0(void)
 {
        return cpus_have_const_cap(ARM64_HAS_32BIT_EL0);

diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h
index 4b39293d0f72dddb809c734aace5df8f80fdc669..d45b42295254d9ab4ad7da44e0070fb8732420cc 100644 (file)
--- a/arch/arm64/include/asm/insn.h
+++ b/arch/arm64/include/asm/insn.h
@@ -65,6 +65,7 @@ enum aarch64_insn_hint_cr_op {
        AARCH64_INSN_HINT_PSB  = 0x11 << 5,
        AARCH64_INSN_HINT_TSB  = 0x12 << 5,
        AARCH64_INSN_HINT_CSDB = 0x14 << 5,
+       AARCH64_INSN_HINT_CLEARBHB = 0x16 << 5,
 
        AARCH64_INSN_HINT_BTI   = 0x20 << 5,
        AARCH64_INSN_HINT_BTIC  = 0x22 << 5,

diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
index c34cd44637d2328d4701d1d5fd2c3500ab295815..ada24a20a5671db51522e6fcdbdd9e7c3b3edf59 100644 (file)
--- a/arch/arm64/include/asm/kvm_asm.h
+++ b/arch/arm64/include/asm/kvm_asm.h
@@ -37,6 +37,7 @@
 #define __SMCCC_WORKAROUND_1_SMC_SZ 36
 #define __SMCCC_WORKAROUND_3_SMC_SZ 36
 #define __SPECTRE_BHB_LOOP_SZ       44
+#define __SPECTRE_BHB_CLEARBHB_SZ   12
 
 #define KVM_HOST_SMCCC_ID(id)                                          \
        ARM_SMCCC_CALL_VAL(ARM_SMCCC_FAST_CALL,                         \
@@ -205,6 +206,7 @@ extern char __smccc_workaround_3_smc[__SMCCC_WORKAROUND_3_SMC_SZ];
 extern char __spectre_bhb_loop_k8[__SPECTRE_BHB_LOOP_SZ];
 extern char __spectre_bhb_loop_k24[__SPECTRE_BHB_LOOP_SZ];
 extern char __spectre_bhb_loop_k32[__SPECTRE_BHB_LOOP_SZ];
+extern char __spectre_bhb_clearbhb[__SPECTRE_BHB_LOOP_SZ];
 
 /*
  * Obtain the PC-relative address of a kernel symbol

diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index f74918d247a5f8f1cc1bca1aa855ad3f7b000bab..1f2209ad2cca177628b48e0814a557b25375f4c1 100644 (file)
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -689,6 +689,7 @@
 #define ID_AA64ISAR1_GPI_IMP_DEF               0x1
 
 /* id_aa64isar2 */
+#define ID_AA64ISAR2_CLEARBHB_SHIFT    28
 #define ID_AA64ISAR2_RPRES_SHIFT       4
 #define ID_AA64ISAR2_WFXT_SHIFT                0
 

diff --git a/arch/arm64/include/asm/vectors.h b/arch/arm64/include/asm/vectors.h
index 1f65c37dc653bfdfd3daca6c6a605974b0de628b..f64613a96d5300de67dba128d13e091298967512 100644 (file)
--- a/arch/arm64/include/asm/vectors.h
+++ b/arch/arm64/include/asm/vectors.h
@@ -32,6 +32,12 @@ enum arm64_bp_harden_el1_vectors {
         * canonical vectors.
         */
        EL1_VECTOR_BHB_FW,
+
+       /*
+        * Use the ClearBHB instruction, before branching to the canonical
+        * vectors.
+        */
+       EL1_VECTOR_BHB_CLEAR_INSN,
 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
 
        /*
@@ -43,6 +49,7 @@ enum arm64_bp_harden_el1_vectors {
 #ifndef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
 #define EL1_VECTOR_BHB_LOOP            -1
 #define EL1_VECTOR_BHB_FW              -1
+#define EL1_VECTOR_BHB_CLEAR_INSN      -1
 #endif /* !CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
 
 /* The vectors to use on return from EL0. e.g. to remap the kernel */

diff --git a/arch/arm64/kernel/cpufeature.c b/arch/arm64/kernel/cpufeature.c
index cad315e1259429088a4842ba648475d6aac339d1..c9108ed406458172431ff4055f12136b73e1ecdf 100644 (file)
--- a/arch/arm64/kernel/cpufeature.c
+++ b/arch/arm64/kernel/cpufeature.c
@@ -211,6 +211,7 @@ static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
 };
 
 static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
+       ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64ISAR2_CLEARBHB_SHIFT, 4, 0),
        ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_RPRES_SHIFT, 4, 0),
        ARM64_FTR_END,
 };

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 34aa2440a95207dee6023557e090272398a2bc78..d5bc1dbdd2fda84cd1e6e9508c07be5b41fae793 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -827,6 +827,7 @@ alternative_else_nop_endif
 #define BHB_MITIGATION_NONE    0
 #define BHB_MITIGATION_LOOP    1
 #define BHB_MITIGATION_FW      2
+#define BHB_MITIGATION_INSN    3
 
        .macro tramp_ventry, vector_start, regsize, kpti, bhb
        .align  7
@@ -843,6 +844,11 @@ alternative_else_nop_endif
        __mitigate_spectre_bhb_loop     x30
        .endif // \bhb == BHB_MITIGATION_LOOP
 
+       .if     \bhb == BHB_MITIGATION_INSN
+       clearbhb
+       isb
+       .endif // \bhb == BHB_MITIGATION_INSN
+
        .if     \kpti == 1
        /*
         * Defend against branch aliasing attacks by pushing a dummy
@@ -919,6 +925,7 @@ SYM_CODE_START_NOALIGN(tramp_vectors)
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_LOOP
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_FW
+       generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_INSN
 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_NONE
 SYM_CODE_END(tramp_vectors)
@@ -981,6 +988,7 @@ SYM_CODE_START(__bp_harden_el1_vectors)
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
        generate_el1_vector     bhb=BHB_MITIGATION_LOOP
        generate_el1_vector     bhb=BHB_MITIGATION_FW
+       generate_el1_vector     bhb=BHB_MITIGATION_INSN
 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */
 SYM_CODE_END(__bp_harden_el1_vectors)
        .popsection

diff --git a/arch/arm64/kernel/proton-pack.c b/arch/arm64/kernel/proton-pack.c
index 47c96944466b063c9319b686bd627b937a2d197b..0e5c411c81aa01873f3f4f8d4a089fc5b28deea9 100644 (file)
--- a/arch/arm64/kernel/proton-pack.c
+++ b/arch/arm64/kernel/proton-pack.c
@@ -824,6 +824,7 @@ int arch_prctl_spec_ctrl_get(struct task_struct *task, unsigned long which)
  * - Mitigated by a branchy loop a CPU specific number of times, and listed
  *   in our "loop mitigated list".
  * - Mitigated in software by the firmware Spectre v2 call.
+ * - Has the ClearBHB instruction to perform the mitigation.
  * - Has the 'Exception Clears Branch History Buffer' (ECBHB) feature, so no
  *   software mitigation in the vectors is needed.
  * - Has CSV2.3, so is unaffected.
@@ -949,6 +950,9 @@ bool is_spectre_bhb_affected(const struct arm64_cpu_capabilities *entry,
        if (supports_csv2p3(scope))
                return false;
 
+       if (supports_clearbhb(scope))
+               return true;
+
        if (spectre_bhb_loop_affected(scope))
                return true;
 
@@ -987,6 +991,8 @@ static int kvm_bhb_get_vecs_size(const char *start)
                 start == __spectre_bhb_loop_k24 ||
                 start == __spectre_bhb_loop_k32)
                return __SPECTRE_BHB_LOOP_SZ;
+       else if (start == __spectre_bhb_clearbhb)
+               return __SPECTRE_BHB_CLEARBHB_SZ;
 
        return 0;
 }
@@ -1027,6 +1033,7 @@ static void kvm_setup_bhb_slot(const char *hyp_vecs_start)
 #define __spectre_bhb_loop_k8 NULL
 #define __spectre_bhb_loop_k24 NULL
 #define __spectre_bhb_loop_k32 NULL
+#define __spectre_bhb_clearbhb NULL
 
 static void kvm_setup_bhb_slot(const char *hyp_vecs_start) { }
 #endif /* CONFIG_KVM */
@@ -1045,6 +1052,11 @@ void spectre_bhb_enable_mitigation(const struct arm64_cpu_capabilities *entry)
        } else if (cpu_mitigations_off()) {
                pr_info_once("spectre-bhb mitigation disabled by command line option\n");
        } else if (supports_ecbhb(SCOPE_LOCAL_CPU)) {
+               state = SPECTRE_MITIGATED;
+       } else if (supports_clearbhb(SCOPE_LOCAL_CPU)) {
+               kvm_setup_bhb_slot(__spectre_bhb_clearbhb);
+               this_cpu_set_vectors(EL1_VECTOR_BHB_CLEAR_INSN);
+
                state = SPECTRE_MITIGATED;
        } else if (spectre_bhb_loop_affected(SCOPE_LOCAL_CPU)) {
                switch (spectre_bhb_loop_affected(SCOPE_SYSTEM)) {

diff --git a/arch/arm64/kvm/hyp/smccc_wa.S b/arch/arm64/kvm/hyp/smccc_wa.S
index 6985dfbc1be19c041191eaa908358de3b5f3f76c..24b281912463dae112636a61a9ba5ef6e4f4879c 100644 (file)
--- a/arch/arm64/kvm/hyp/smccc_wa.S
+++ b/arch/arm64/kvm/hyp/smccc_wa.S
@@ -96,3 +96,12 @@ SYM_DATA_START(__spectre_bhb_loop_k32)
 1:     .org __spectre_bhb_loop_k32 + __SPECTRE_BHB_LOOP_SZ
        .org 1b
 SYM_DATA_END(__spectre_bhb_loop_k32)
+
+       .global __spectre_bhb_clearbhb
+SYM_DATA_START(__spectre_bhb_clearbhb)
+       esb
+       clearbhb
+       isb
+1:     .org __spectre_bhb_clearbhb + __SPECTRE_BHB_CLEARBHB_SZ
+       .org 1b
+SYM_DATA_END(__spectre_bhb_clearbhb)