powerpc/mm: Fixup tlbie vs store ordering issue on POWER9

On POWER9, under some circumstances, a broadcast TLB invalidation
might complete before all previous stores have drained, potentially
allowing stale stores from becoming visible after the invalidation.
This works around it by doubling up those TLB invalidations which was
verified by HW to be sufficient to close the risk window.

This will be documented in a yet-to-be-published errata.

Fixes: 1a472c9dba6b ("powerpc/mm/radix: Add tlbflush routines")
Signed-off-by: Aneesh Kumar K.V <aneesh.kumar@linux.vnet.ibm.com>
[mpe: Enable the feature in the DT CPU features code for all Power9,
      rename the feature to CPU_FTR_P9_TLBIE_BUG per benh.]
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h
index a2c5c95882cf08d3fda4b4c87cc13d73ba66a652..2e2bacbdf6ed486f1aad88f3a7f3c73c2fd898df 100644 (file)
--- a/arch/powerpc/include/asm/cputable.h
+++ b/arch/powerpc/include/asm/cputable.h
@@ -203,6 +203,7 @@ static inline void cpu_feature_keys_init(void) { }
 #define CPU_FTR_DAWR                   LONG_ASM_CONST(0x0400000000000000)
 #define CPU_FTR_DABRX                  LONG_ASM_CONST(0x0800000000000000)
 #define CPU_FTR_PMAO_BUG               LONG_ASM_CONST(0x1000000000000000)
+#define CPU_FTR_P9_TLBIE_BUG           LONG_ASM_CONST(0x2000000000000000)
 #define CPU_FTR_POWER9_DD1             LONG_ASM_CONST(0x4000000000000000)
 #define CPU_FTR_POWER9_DD2_1           LONG_ASM_CONST(0x8000000000000000)
 
@@ -465,7 +466,7 @@ static inline void cpu_feature_keys_init(void) { }
            CPU_FTR_CFAR | CPU_FTR_HVMODE | CPU_FTR_VMX_COPY | \
            CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \
            CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | \
-           CPU_FTR_PKEY)
+           CPU_FTR_PKEY | CPU_FTR_P9_TLBIE_BUG)
 #define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \
                             (~CPU_FTR_SAO))
 #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9

diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c
index 0bcfb0f256e107e133d7f0ebf8c19c9d8438fd7f..8ca5d5b74618371904ec7af9c436050ea35cdf0e 100644 (file)
--- a/arch/powerpc/kernel/dt_cpu_ftrs.c
+++ b/arch/powerpc/kernel/dt_cpu_ftrs.c
@@ -709,6 +709,9 @@ static __init void cpufeatures_cpu_quirks(void)
                cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1;
        else if ((version & 0xffffefff) == 0x004e0201)
                cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1;
+
+       if ((version & 0xffff0000) == 0x004e0000)
+               cur_cpu_spec->cpu_features |= CPU_FTR_P9_TLBIE_BUG;
 }
 
 static void __init cpufeatures_setup_finished(void)

diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c
index 0c854816e653e25238f87c1cf9c44a8ed911df44..0837b9738d764c8e949902aecb02311443ffd626 100644 (file)
--- a/arch/powerpc/kvm/book3s_64_mmu_radix.c
+++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c
@@ -157,6 +157,9 @@ static void kvmppc_radix_tlbie_page(struct kvm *kvm, unsigned long addr,
        asm volatile("ptesync": : :"memory");
        asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
                     : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
+       if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG))
+               asm volatile(PPC_TLBIE_5(%0, %1, 0, 0, 1)
+                            : : "r" (addr), "r" (kvm->arch.lpid) : "memory");
        asm volatile("ptesync": : :"memory");
 }
 

diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 8888e625a9991c4ab3dea1e05f1a6aa8508b30e1..e1c083fbe43480d0b0990ce25f9cd6f52dce2bac 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -473,6 +473,17 @@ static void do_tlbies(struct kvm *kvm, unsigned long *rbvalues,
                        trace_tlbie(kvm->arch.lpid, 0, rbvalues[i],
                                kvm->arch.lpid, 0, 0, 0);
                }
+
+               if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+                       /*
+                        * Need the extra ptesync to make sure we don't
+                        * re-order the tlbie
+                        */
+                       asm volatile("ptesync": : :"memory");
+                       asm volatile(PPC_TLBIE_5(%0,%1,0,0,0) : :
+                                    "r" (rbvalues[0]), "r" (kvm->arch.lpid));
+               }
+
                asm volatile("eieio; tlbsync; ptesync" : : : "memory");
                kvm->arch.tlbie_lock = 0;
        } else {

diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c
index a0675e91ad7d11318d8f630a378509e5703dfe70..656933c859256bff1c9f1848830c0616033218aa 100644 (file)
--- a/arch/powerpc/mm/hash_native_64.c
+++ b/arch/powerpc/mm/hash_native_64.c
@@ -201,6 +201,15 @@ static inline unsigned long  ___tlbie(unsigned long vpn, int psize,
        return va;
 }
 
+static inline void fixup_tlbie(unsigned long vpn, int psize, int apsize, int ssize)
+{
+       if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+               /* Need the extra ptesync to ensure we don't reorder tlbie*/
+               asm volatile("ptesync": : :"memory");
+               ___tlbie(vpn, psize, apsize, ssize);
+       }
+}
+
 static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize)
 {
        unsigned long rb;
@@ -278,6 +287,7 @@ static inline void tlbie(unsigned long vpn, int psize, int apsize,
                asm volatile("ptesync": : :"memory");
        } else {
                __tlbie(vpn, psize, apsize, ssize);
+               fixup_tlbie(vpn, psize, apsize, ssize);
                asm volatile("eieio; tlbsync; ptesync": : :"memory");
        }
        if (lock_tlbie && !use_local)
@@ -771,7 +781,7 @@ static void native_hpte_clear(void)
  */
 static void native_flush_hash_range(unsigned long number, int local)
 {
-       unsigned long vpn;
+       unsigned long vpn = 0;
        unsigned long hash, index, hidx, shift, slot;
        struct hash_pte *hptep;
        unsigned long hpte_v;
@@ -843,6 +853,10 @@ static void native_flush_hash_range(unsigned long number, int local)
                                __tlbie(vpn, psize, psize, ssize);
                        } pte_iterate_hashed_end();
                }
+               /*
+                * Just do one more with the last used values.
+                */
+               fixup_tlbie(vpn, psize, psize, ssize);
                asm volatile("eieio; tlbsync; ptesync":::"memory");
 
                if (lock_tlbie)

diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c
index 28c980eb4422284d788716e245934c679925ad86..adf469f312f2044b9b4035a201b63b3da393e6ce 100644 (file)
--- a/arch/powerpc/mm/pgtable_64.c
+++ b/arch/powerpc/mm/pgtable_64.c
@@ -481,6 +481,7 @@ void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
                             "r" (TLBIEL_INVAL_SET_LPID), "r" (lpid));
                trace_tlbie(lpid, 0, TLBIEL_INVAL_SET_LPID, lpid, 2, 0, 0);
        }
+       /* do we need fixup here ?*/
        asm volatile("eieio; tlbsync; ptesync" : : : "memory");
 }
 EXPORT_SYMBOL_GPL(mmu_partition_table_set_entry);

diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 74354c26d31678bfa35a81fa1a6a31fb5914fe71..a07f5372a4bf36ce726408891defc380dd9fea59 100644 (file)
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -151,6 +151,17 @@ static inline void __tlbie_va(unsigned long va, unsigned long pid,
        trace_tlbie(0, 0, rb, rs, ric, prs, r);
 }
 
+static inline void fixup_tlbie(void)
+{
+       unsigned long pid = 0;
+       unsigned long va = ((1UL << 52) - 1);
+
+       if (cpu_has_feature(CPU_FTR_P9_TLBIE_BUG)) {
+               asm volatile("ptesync": : :"memory");
+               __tlbie_va(va, pid, mmu_get_ap(MMU_PAGE_64K), RIC_FLUSH_TLB);
+       }
+}
+
 /*
  * We use 128 set in radix mode and 256 set in hpt mode.
  */
@@ -200,6 +211,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned long ric)
        default:
                __tlbie_pid(pid, RIC_FLUSH_ALL);
        }
+       fixup_tlbie();
        asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
@@ -253,6 +265,7 @@ static inline void _tlbie_va(unsigned long va, unsigned long pid,
 
        asm volatile("ptesync": : :"memory");
        __tlbie_va(va, pid, ap, ric);
+       fixup_tlbie();
        asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
@@ -264,6 +277,7 @@ static inline void _tlbie_va_range(unsigned long start, unsigned long end,
        if (also_pwc)
                __tlbie_pid(pid, RIC_FLUSH_PWC);
        __tlbie_va_range(start, end, pid, page_size, psize);
+       fixup_tlbie();
        asm volatile("eieio; tlbsync; ptesync": : :"memory");
 }
 
@@ -498,6 +512,7 @@ void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
                        if (hflush)
                                __tlbie_va_range(hstart, hend, pid,
                                                HPAGE_PMD_SIZE, MMU_PAGE_2M);
+                       fixup_tlbie();
                        asm volatile("eieio; tlbsync; ptesync": : :"memory");
                }
        }