KVM: PPC: Book3S HV: Fix TLB management on SMT8 POWER9 and POWER10 processors

[ Upstream commit 77bbbc0cf84834ed130838f7ac1988567f4d0288 ]

The POWER9 vCPU TLB management code assumes all threads in a core share
a TLB, and that TLBIEL execued by one thread will invalidate TLBs for
all threads. This is not the case for SMT8 capable POWER9 and POWER10
(big core) processors, where the TLB is split between groups of threads.
This results in TLB multi-hits, random data corruption, etc.

Fix this by introducing cpu_first_tlb_thread_sibling etc., to determine
which siblings share TLBs, and use that in the guest TLB flushing code.

[npiggin@gmail.com: add changelog and comment]

Signed-off-by: Paul Mackerras <paulus@ozlabs.org>
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Reviewed-by: Fabiano Rosas <farosas@linux.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>
Link: https://lore.kernel.org/r/20210602040441.3984352-1-npiggin@gmail.com
Signed-off-by: Sasha Levin <sashal@kernel.org>

diff --git a/arch/powerpc/include/asm/cputhreads.h b/arch/powerpc/include/asm/cputhreads.h
index deb99fd6e060fe9b098fabb826e3323d49b07c3c..51f533b6ece02a057229083b8e962d40648e3c75 100644 (file)
--- a/arch/powerpc/include/asm/cputhreads.h
+++ b/arch/powerpc/include/asm/cputhreads.h
@@ -99,6 +99,36 @@ static inline int cpu_last_thread_sibling(int cpu)
        return cpu | (threads_per_core - 1);
 }
 
+/*
+ * tlb_thread_siblings are siblings which share a TLB. This is not
+ * architected, is not something a hypervisor could emulate and a future
+ * CPU may change behaviour even in compat mode, so this should only be
+ * used on PowerNV, and only with care.
+ */
+static inline int cpu_first_tlb_thread_sibling(int cpu)
+{
+       if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8))
+               return cpu & ~0x6;      /* Big Core */
+       else
+               return cpu_first_thread_sibling(cpu);
+}
+
+static inline int cpu_last_tlb_thread_sibling(int cpu)
+{
+       if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8))
+               return cpu | 0x6;       /* Big Core */
+       else
+               return cpu_last_thread_sibling(cpu);
+}
+
+static inline int cpu_tlb_thread_sibling_step(void)
+{
+       if (cpu_has_feature(CPU_FTR_ARCH_300) && (threads_per_core == 8))
+               return 2;               /* Big Core */
+       else
+               return 1;
+}
+
 static inline u32 get_tensr(void)
 {
 #ifdef CONFIG_BOOKE

diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c
index 6938b793a015091bdf079625fb5555bcbd6f76b3..9011857c0434d5acc7f416596e931891ce78e7d2 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv.c
+++ b/arch/powerpc/kvm/book3s_hv.c
@@ -2536,7 +2536,7 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
        cpumask_t *cpu_in_guest;
        int i;
 
-       cpu = cpu_first_thread_sibling(cpu);
+       cpu = cpu_first_tlb_thread_sibling(cpu);
        if (nested) {
                cpumask_set_cpu(cpu, &nested->need_tlb_flush);
                cpu_in_guest = &nested->cpu_in_guest;
@@ -2550,9 +2550,10 @@ static void radix_flush_cpu(struct kvm *kvm, int cpu, struct kvm_vcpu *vcpu)
         * the other side is the first smp_mb() in kvmppc_run_core().
         */
        smp_mb();
-       for (i = 0; i < threads_per_core; ++i)
-               if (cpumask_test_cpu(cpu + i, cpu_in_guest))
-                       smp_call_function_single(cpu + i, do_nothing, NULL, 1);
+       for (i = cpu; i <= cpu_last_tlb_thread_sibling(cpu);
+                                       i += cpu_tlb_thread_sibling_step())
+               if (cpumask_test_cpu(i, cpu_in_guest))
+                       smp_call_function_single(i, do_nothing, NULL, 1);
 }
 
 static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
@@ -2583,8 +2584,8 @@ static void kvmppc_prepare_radix_vcpu(struct kvm_vcpu *vcpu, int pcpu)
         */
        if (prev_cpu != pcpu) {
                if (prev_cpu >= 0 &&
-                   cpu_first_thread_sibling(prev_cpu) !=
-                   cpu_first_thread_sibling(pcpu))
+                   cpu_first_tlb_thread_sibling(prev_cpu) !=
+                   cpu_first_tlb_thread_sibling(pcpu))
                        radix_flush_cpu(kvm, prev_cpu, vcpu);
                if (nested)
                        nested->prev_cpu[vcpu->arch.nested_vcpu_id] = pcpu;

diff --git a/arch/powerpc/kvm/book3s_hv_builtin.c b/arch/powerpc/kvm/book3s_hv_builtin.c
index 7c1909657b556f58bb10dde3b8515d48b6618c00..4a91b543a8540357eec7a1c6760a43f1e17df362 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_builtin.c
+++ b/arch/powerpc/kvm/book3s_hv_builtin.c
@@ -847,7 +847,7 @@ void kvmppc_check_need_tlb_flush(struct kvm *kvm, int pcpu,
         * Thus we make all 4 threads use the same bit.
         */
        if (cpu_has_feature(CPU_FTR_ARCH_300))
-               pcpu = cpu_first_thread_sibling(pcpu);
+               pcpu = cpu_first_tlb_thread_sibling(pcpu);
 
        if (nested)
                need_tlb_flush = &nested->need_tlb_flush;

diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
index 220305454c23c9e8e8a2b3d975682de53615c93f..9bf3be438ac58d08f7b248b6b57e468baa18e6c1 100644 (file)
--- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c
+++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c
@@ -67,7 +67,7 @@ static int global_invalidates(struct kvm *kvm)
                 * so use the bit for the first thread to represent the core.
                 */
                if (cpu_has_feature(CPU_FTR_ARCH_300))
-                       cpu = cpu_first_thread_sibling(cpu);
+                       cpu = cpu_first_tlb_thread_sibling(cpu);
                cpumask_clear_cpu(cpu, &kvm->arch.need_tlb_flush);
        }