arm64: errata: Work around AmpereOne's erratum AC04_CPU_23

On AmpereOne AC04, updates to HCR_EL2 can rarely corrupt simultaneous
translations for data addresses initiated by load/store instructions.
Only instruction initiated translations are vulnerable, not translations
from prefetches for example. A DSB before the store to HCR_EL2 is
sufficient to prevent older instructions from hitting the window for
corruption, and an ISB after is sufficient to prevent younger
instructions from hitting the window for corruption.

Signed-off-by: D Scott Phillips <scott@os.amperecomputing.com>
Reviewed-by: Oliver Upton <oliver.upton@linux.dev>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20250513184514.2678288-1-scott@os.amperecomputing.com
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/Documentation/arch/arm64/silicon-errata.rst b/Documentation/arch/arm64/silicon-errata.rst
index f968c13b46a78701b657c5e29610a708841e1574..b18ef4064bc04672e31d3a53d42ae6055deb12f2 100644 (file)
--- a/Documentation/arch/arm64/silicon-errata.rst
+++ b/Documentation/arch/arm64/silicon-errata.rst
@@ -57,6 +57,8 @@ stable kernels.
 +----------------+-----------------+-----------------+-----------------------------+
 | Ampere         | AmpereOne AC04  | AC04_CPU_10     | AMPERE_ERRATUM_AC03_CPU_38  |
 +----------------+-----------------+-----------------+-----------------------------+
+| Ampere         | AmpereOne AC04  | AC04_CPU_23     | AMPERE_ERRATUM_AC04_CPU_23  |
++----------------+-----------------+-----------------+-----------------------------+
 +----------------+-----------------+-----------------+-----------------------------+
 | ARM            | Cortex-A510     | #2457168        | ARM64_ERRATUM_2457168       |
 +----------------+-----------------+-----------------+-----------------------------+

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index a182295e6f08bfa0f3e6f630dc4adfe797a4d273..3ae4e80e3002b2500e7276946383bb35ffb37c70 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -464,6 +464,23 @@ config AMPERE_ERRATUM_AC03_CPU_38
 
          If unsure, say Y.
 
+config AMPERE_ERRATUM_AC04_CPU_23
+        bool "AmpereOne: AC04_CPU_23:  Failure to synchronize writes to HCR_EL2 may corrupt address translations."
+       default y
+       help
+         This option adds an alternative code sequence to work around Ampere
+         errata AC04_CPU_23 on AmpereOne.
+
+         Updates to HCR_EL2 can rarely corrupt simultaneous translations for
+         data addresses initiated by load/store instructions. Only
+         instruction initiated translations are vulnerable, not translations
+         from prefetches for example. A DSB before the store to HCR_EL2 is
+         sufficient to prevent older instructions from hitting the window
+         for corruption, and an ISB after is sufficient to prevent younger
+         instructions from hitting the window for corruption.
+
+         If unsure, say Y.
+
 config ARM64_WORKAROUND_CLEAN_CACHE
        bool
 

diff --git a/arch/arm64/include/asm/el2_setup.h b/arch/arm64/include/asm/el2_setup.h
index ebceaae3c749b84395c9c5eccf0caf874697ad11..2500fd0a1f66a872ece7d29a50527e1f54ce799e 100644 (file)
--- a/arch/arm64/include/asm/el2_setup.h
+++ b/arch/arm64/include/asm/el2_setup.h
@@ -38,7 +38,7 @@
 
        orr     x0, x0, #HCR_E2H
 .LnVHE_\@:
-       msr     hcr_el2, x0
+       msr_hcr_el2 x0
        isb
 .endm
 

diff --git a/arch/arm64/include/asm/hardirq.h b/arch/arm64/include/asm/hardirq.h
index cbfa7b6f2e098f3d734d43d30ce77309349a28d6..77d6b8c63d4e603d832844d28ee04b2d3d5fd76c 100644 (file)
--- a/arch/arm64/include/asm/hardirq.h
+++ b/arch/arm64/include/asm/hardirq.h
@@ -41,7 +41,7 @@ do {                                                                  \
                                                                        \
        ___hcr = read_sysreg(hcr_el2);                                  \
        if (!(___hcr & HCR_TGE)) {                                      \
-               write_sysreg(___hcr | HCR_TGE, hcr_el2);                \
+               write_sysreg_hcr(___hcr | HCR_TGE);                     \
                isb();                                                  \
        }                                                               \
        /*                                                              \
@@ -82,7 +82,7 @@ do {                                                                  \
         */                                                             \
        barrier();                                                      \
        if (!___ctx->cnt && !(___hcr & HCR_TGE))                        \
-               write_sysreg(___hcr, hcr_el2);                          \
+               write_sysreg_hcr(___hcr);                               \
 } while (0)
 
 static inline void ack_bad_irq(unsigned int irq)

diff --git a/arch/arm64/include/asm/sysreg.h b/arch/arm64/include/asm/sysreg.h
index 2639d3633073de10f5040a7efff059021f847530..7284828f0dc9eca0eabd7aecd2414a445c5ccc0b 100644 (file)
--- a/arch/arm64/include/asm/sysreg.h
+++ b/arch/arm64/include/asm/sysreg.h
@@ -1091,6 +1091,15 @@
        __emit_inst(0xd5000000|(\sreg)|(.L__gpr_num_\rt))
        .endm
 
+       .macro  msr_hcr_el2, reg
+#if IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23)
+       dsb     nsh
+       msr     hcr_el2, \reg
+       isb
+#else
+       msr     hcr_el2, \reg
+#endif
+       .endm
 #else
 
 #include <linux/bitfield.h>
@@ -1178,6 +1187,13 @@
                write_sysreg(__scs_new, sysreg);                        \
 } while (0)
 
+#define sysreg_clear_set_hcr(clear, set) do {                          \
+       u64 __scs_val = read_sysreg(hcr_el2);                           \
+       u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set);            \
+       if (__scs_new != __scs_val)                                     \
+               write_sysreg_hcr(__scs_new);                    \
+} while (0)
+
 #define sysreg_clear_set_s(sysreg, clear, set) do {                    \
        u64 __scs_val = read_sysreg_s(sysreg);                          \
        u64 __scs_new = (__scs_val & ~(u64)(clear)) | (set);            \
@@ -1185,6 +1201,17 @@
                write_sysreg_s(__scs_new, sysreg);                      \
 } while (0)
 
+#define write_sysreg_hcr(__val) do {                                   \
+       if (IS_ENABLED(CONFIG_AMPERE_ERRATUM_AC04_CPU_23) &&            \
+          (!system_capabilities_finalized() ||                         \
+           alternative_has_cap_unlikely(ARM64_WORKAROUND_AMPERE_AC04_CPU_23))) \
+               asm volatile("dsb nsh; msr hcr_el2, %x0; isb"           \
+                            : : "rZ" (__val));                         \
+       else                                                            \
+               asm volatile("msr hcr_el2, %x0"                         \
+                            : : "rZ" (__val));                         \
+} while (0)
+
 #define read_sysreg_par() ({                                           \
        u64 par;                                                        \
        asm(ALTERNATIVE("nop", "dmb sy", ARM64_WORKAROUND_1508412));    \

diff --git a/arch/arm64/kernel/cpu_errata.c b/arch/arm64/kernel/cpu_errata.c
index b55f5f7057502c641fbd096c0f9b6e6ed32b7dec..60f1b70fc0845f267432a5378d874be2c2a7eace 100644 (file)
--- a/arch/arm64/kernel/cpu_errata.c
+++ b/arch/arm64/kernel/cpu_errata.c
@@ -557,6 +557,13 @@ static const struct midr_range erratum_ac03_cpu_38_list[] = {
 };
 #endif
 
+#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
+static const struct midr_range erratum_ac04_cpu_23_list[] = {
+       MIDR_ALL_VERSIONS(MIDR_AMPERE1A),
+       {},
+};
+#endif
+
 const struct arm64_cpu_capabilities arm64_errata[] = {
 #ifdef CONFIG_ARM64_WORKAROUND_CLEAN_CACHE
        {
@@ -875,6 +882,13 @@ const struct arm64_cpu_capabilities arm64_errata[] = {
                .capability = ARM64_WORKAROUND_AMPERE_AC03_CPU_38,
                ERRATA_MIDR_RANGE_LIST(erratum_ac03_cpu_38_list),
        },
+#endif
+#ifdef CONFIG_AMPERE_ERRATUM_AC04_CPU_23
+       {
+               .desc = "AmpereOne erratum AC04_CPU_23",
+               .capability = ARM64_WORKAROUND_AMPERE_AC04_CPU_23,
+               ERRATA_MIDR_RANGE_LIST(erratum_ac04_cpu_23_list),
+       },
 #endif
        {
                .desc = "Broken CNTVOFF_EL2",

diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S
index ae990da1eae5a97221b87fdd4b839eccfeaa125c..36e2d26b54f5c2225c72caf75a2da96bc39d282f 100644 (file)
--- a/arch/arm64/kernel/hyp-stub.S
+++ b/arch/arm64/kernel/hyp-stub.S
@@ -97,7 +97,7 @@ SYM_CODE_START_LOCAL(__finalise_el2)
 2:
        // Engage the VHE magic!
        mov_q   x0, HCR_HOST_VHE_FLAGS
-       msr     hcr_el2, x0
+       msr_hcr_el2 x0
        isb
 
        // Use the EL1 allocated stack, per-cpu offset

diff --git a/arch/arm64/kvm/at.c b/arch/arm64/kvm/at.c
index f74a66ce3064b7a39905dfc9af10c4cb260e96f1..9c13e70fadf5e28cb840cad09d87324ade3d1dc7 100644 (file)
--- a/arch/arm64/kvm/at.c
+++ b/arch/arm64/kvm/at.c
@@ -516,7 +516,7 @@ static void __mmu_config_save(struct mmu_config *config)
 
 static void __mmu_config_restore(struct mmu_config *config)
 {
-       write_sysreg(config->hcr,       hcr_el2);
+       write_sysreg_hcr(config->hcr);
 
        /*
         * ARM errata 1165522 and 1530923 require TGE to be 1 before
@@ -1267,7 +1267,7 @@ static u64 __kvm_at_s1e01_fast(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
 
 skip_mmu_switch:
        /* Clear TGE, enable S2 translation, we're rolling */
-       write_sysreg((config.hcr & ~HCR_TGE) | HCR_VM,  hcr_el2);
+       write_sysreg_hcr((config.hcr & ~HCR_TGE) | HCR_VM);
        isb();
 
        switch (op) {
@@ -1350,7 +1350,7 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
                if (!vcpu_el2_e2h_is_set(vcpu))
                        val |= HCR_NV | HCR_NV1;
 
-               write_sysreg(val, hcr_el2);
+               write_sysreg_hcr(val);
                isb();
 
                par = SYS_PAR_EL1_F;
@@ -1375,7 +1375,7 @@ void __kvm_at_s1e2(struct kvm_vcpu *vcpu, u32 op, u64 vaddr)
                if (!fail)
                        par = read_sysreg_par();
 
-               write_sysreg(hcr, hcr_el2);
+               write_sysreg_hcr(hcr);
                isb();
        }
 

diff --git a/arch/arm64/kvm/hyp/include/hyp/switch.h b/arch/arm64/kvm/hyp/include/hyp/switch.h
index b741ea6aefa58f5dabf02aeb7f9bd8029db151be..06aa37dbc957d9733e070077b23885a0df2e3368 100644 (file)
--- a/arch/arm64/kvm/hyp/include/hyp/switch.h
+++ b/arch/arm64/kvm/hyp/include/hyp/switch.h
@@ -301,7 +301,7 @@ static inline void ___activate_traps(struct kvm_vcpu *vcpu, u64 hcr)
        if (cpus_have_final_cap(ARM64_WORKAROUND_CAVIUM_TX2_219_TVM))
                hcr |= HCR_TVM;
 
-       write_sysreg(hcr, hcr_el2);
+       write_sysreg_hcr(hcr);
 
        if (cpus_have_final_cap(ARM64_HAS_RAS_EXTN) && (hcr & HCR_VSE))
                write_sysreg_s(vcpu->arch.vsesr_el2, SYS_VSESR_EL2);

diff --git a/arch/arm64/kvm/hyp/nvhe/host.S b/arch/arm64/kvm/hyp/nvhe/host.S
index 58f0cb2298cc28c043b740072efcc63f24d6951a..eef15b374abb0d0f5744d8693e6ffdba5e4a5e55 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/host.S
+++ b/arch/arm64/kvm/hyp/nvhe/host.S
@@ -124,7 +124,7 @@ SYM_FUNC_START(__hyp_do_panic)
        /* Ensure host stage-2 is disabled */
        mrs     x0, hcr_el2
        bic     x0, x0, #HCR_VM
-       msr     hcr_el2, x0
+       msr_hcr_el2 x0
        isb
        tlbi    vmalls12e1
        dsb     nsh

diff --git a/arch/arm64/kvm/hyp/nvhe/hyp-init.S b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
index f8af11189572fce7e6ab153e45ae745b611edb51..aada42522e7be36371a1a48cc5eaff8972c02436 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/hyp-init.S
+++ b/arch/arm64/kvm/hyp/nvhe/hyp-init.S
@@ -100,7 +100,7 @@ SYM_CODE_START_LOCAL(___kvm_hyp_init)
        msr     mair_el2, x1
 
        ldr     x1, [x0, #NVHE_INIT_HCR_EL2]
-       msr     hcr_el2, x1
+       msr_hcr_el2 x1
 
        mov     x2, #HCR_E2H
        and     x2, x1, x2
@@ -262,7 +262,7 @@ reset:
 
 alternative_if ARM64_KVM_PROTECTED_MODE
        mov_q   x5, HCR_HOST_NVHE_FLAGS
-       msr     hcr_el2, x5
+       msr_hcr_el2 x5
 alternative_else_nop_endif
 
        /* Install stub vectors */

diff --git a/arch/arm64/kvm/hyp/nvhe/mem_protect.c b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
index 2a5284f749b427927791f0706f66542e90611df9..3aca6278d2f6d311665166a28165455c09134711 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/mem_protect.c
+++ b/arch/arm64/kvm/hyp/nvhe/mem_protect.c
@@ -309,7 +309,7 @@ int __pkvm_prot_finalize(void)
         */
        kvm_flush_dcache_to_poc(params, sizeof(*params));
 
-       write_sysreg(params->hcr_el2, hcr_el2);
+       write_sysreg_hcr(params->hcr_el2);
        __load_stage2(&host_mmu.arch.mmu, &host_mmu.arch);
 
        /*

diff --git a/arch/arm64/kvm/hyp/nvhe/switch.c b/arch/arm64/kvm/hyp/nvhe/switch.c
index 7d2ba6ef026186e2e9f7eae48132c0ede4285c90..4024fafbe3594c7fa1d5a4d3282ac3ad6258b70f 100644 (file)
--- a/arch/arm64/kvm/hyp/nvhe/switch.c
+++ b/arch/arm64/kvm/hyp/nvhe/switch.c
@@ -142,7 +142,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
 
        __deactivate_traps_common(vcpu);
 
-       write_sysreg(this_cpu_ptr(&kvm_init_params)->hcr_el2, hcr_el2);
+       write_sysreg_hcr(this_cpu_ptr(&kvm_init_params)->hcr_el2);
 
        __deactivate_cptr_traps(vcpu);
        write_sysreg(__kvm_hyp_host_vector, vbar_el2);

diff --git a/arch/arm64/kvm/hyp/vgic-v3-sr.c b/arch/arm64/kvm/hyp/vgic-v3-sr.c
index 50aa8dbcae75b059cfbef93ff19adc27e3f9277b..f8a91780e49a94925a495da0f734c2f95811c67a 100644 (file)
--- a/arch/arm64/kvm/hyp/vgic-v3-sr.c
+++ b/arch/arm64/kvm/hyp/vgic-v3-sr.c
@@ -446,7 +446,7 @@ u64 __vgic_v3_get_gic_config(void)
        if (has_vhe()) {
                flags = local_daif_save();
        } else {
-               sysreg_clear_set(hcr_el2, 0, HCR_AMO | HCR_FMO | HCR_IMO);
+               sysreg_clear_set_hcr(0, HCR_AMO | HCR_FMO | HCR_IMO);
                isb();
        }
 
@@ -461,7 +461,7 @@ u64 __vgic_v3_get_gic_config(void)
        if (has_vhe()) {
                local_daif_restore(flags);
        } else {
-               sysreg_clear_set(hcr_el2, HCR_AMO | HCR_FMO | HCR_IMO, 0);
+               sysreg_clear_set_hcr(HCR_AMO | HCR_FMO | HCR_IMO, 0);
                isb();
        }
 

diff --git a/arch/arm64/kvm/hyp/vhe/switch.c b/arch/arm64/kvm/hyp/vhe/switch.c
index 731a0378ed1328c6929c9b82e5d7fbc6e4b2b0d4..faacdfb328af632f859cb08c4fb9ef1a091340c9 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/switch.c
+++ b/arch/arm64/kvm/hyp/vhe/switch.c
@@ -184,7 +184,7 @@ static void __deactivate_traps(struct kvm_vcpu *vcpu)
 
        ___deactivate_traps(vcpu);
 
-       write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
+       write_sysreg_hcr(HCR_HOST_VHE_FLAGS);
 
        if (has_cntpoff()) {
                struct timer_map map;

diff --git a/arch/arm64/kvm/hyp/vhe/tlb.c b/arch/arm64/kvm/hyp/vhe/tlb.c
index 3d50a1bd2bdbcb1d18093b2854435c37dff06fd6..ec256981862977d6f04cb82ba7d6aa4571df331b 100644 (file)
--- a/arch/arm64/kvm/hyp/vhe/tlb.c
+++ b/arch/arm64/kvm/hyp/vhe/tlb.c
@@ -63,7 +63,7 @@ static void enter_vmid_context(struct kvm_s2_mmu *mmu,
        __load_stage2(mmu, mmu->arch);
        val = read_sysreg(hcr_el2);
        val &= ~HCR_TGE;
-       write_sysreg(val, hcr_el2);
+       write_sysreg_hcr(val);
        isb();
 }
 
@@ -73,7 +73,7 @@ static void exit_vmid_context(struct tlb_inv_context *cxt)
         * We're done with the TLB operation, let's restore the host's
         * view of HCR_EL2.
         */
-       write_sysreg(HCR_HOST_VHE_FLAGS, hcr_el2);
+       write_sysreg_hcr(HCR_HOST_VHE_FLAGS);
        isb();
 
        /* ... and the stage-2 MMU context that we switched away from */

diff --git a/arch/arm64/tools/cpucaps b/arch/arm64/tools/cpucaps
index 772c1b008e437ed34cedb1c0f663c4dcea8f6759..72f10b74ce807b035190313845c39a68544151f2 100644 (file)
--- a/arch/arm64/tools/cpucaps
+++ b/arch/arm64/tools/cpucaps
@@ -94,6 +94,7 @@ WORKAROUND_2457168
 WORKAROUND_2645198
 WORKAROUND_2658417
 WORKAROUND_AMPERE_AC03_CPU_38
+WORKAROUND_AMPERE_AC04_CPU_23
 WORKAROUND_TRBE_OVERWRITE_FILL_MODE
 WORKAROUND_TSB_FLUSH_FAILURE
 WORKAROUND_TRBE_WRITE_OUT_OF_RANGE