KVM: arm64: nv: Invalidate TLBs based on shadow S2 TTL-like information

In order to be able to make S2 TLB invalidations more performant on NV,
let's use a scheme derived from the FEAT_TTL extension.

If bits [56:55] in the leaf descriptor translating the address in the
corresponding shadow S2 are non-zero, they indicate a level which can
be used as an invalidation range. This allows further reduction of the
systematic over-invalidation that takes place otherwise.

Signed-off-by: Marc Zyngier <maz@kernel.org>
Link: https://lore.kernel.org/r/20240614144552.2773592-14-maz@kernel.org
Signed-off-by: Oliver Upton <oliver.upton@linux.dev>

diff --git a/arch/arm64/kvm/nested.c b/arch/arm64/kvm/nested.c
index 592241de60e340719d2a57bee1fb783243848151..1860ed4a4c003f84bd39de3bf039157ef21c2a0e 100644 (file)
--- a/arch/arm64/kvm/nested.c
+++ b/arch/arm64/kvm/nested.c
@@ -4,6 +4,7 @@
  * Author: Jintack Lim <jintack.lim@linaro.org>
  */
 
+#include <linux/bitfield.h>
 #include <linux/kvm.h>
 #include <linux/kvm_host.h>
 
@@ -420,12 +421,94 @@ static unsigned int ttl_to_size(u8 ttl)
        return max_size;
 }
 
+/*
+ * Compute the equivalent of the TTL field by parsing the shadow PT.  The
+ * granule size is extracted from the cached VTCR_EL2.TG0 while the level is
+ * retrieved from first entry carrying the level as a tag.
+ */
+static u8 get_guest_mapping_ttl(struct kvm_s2_mmu *mmu, u64 addr)
+{
+       u64 tmp, sz = 0, vtcr = mmu->tlb_vtcr;
+       kvm_pte_t pte;
+       u8 ttl, level;
+
+       lockdep_assert_held_write(&kvm_s2_mmu_to_kvm(mmu)->mmu_lock);
+
+       switch (vtcr & VTCR_EL2_TG0_MASK) {
+       case VTCR_EL2_TG0_4K:
+               ttl = (TLBI_TTL_TG_4K << 2);
+               break;
+       case VTCR_EL2_TG0_16K:
+               ttl = (TLBI_TTL_TG_16K << 2);
+               break;
+       case VTCR_EL2_TG0_64K:
+       default:            /* IMPDEF: treat any other value as 64k */
+               ttl = (TLBI_TTL_TG_64K << 2);
+               break;
+       }
+
+       tmp = addr;
+
+again:
+       /* Iteratively compute the block sizes for a particular granule size */
+       switch (vtcr & VTCR_EL2_TG0_MASK) {
+       case VTCR_EL2_TG0_4K:
+               if      (sz < SZ_4K)    sz = SZ_4K;
+               else if (sz < SZ_2M)    sz = SZ_2M;
+               else if (sz < SZ_1G)    sz = SZ_1G;
+               else                    sz = 0;
+               break;
+       case VTCR_EL2_TG0_16K:
+               if      (sz < SZ_16K)   sz = SZ_16K;
+               else if (sz < SZ_32M)   sz = SZ_32M;
+               else                    sz = 0;
+               break;
+       case VTCR_EL2_TG0_64K:
+       default:            /* IMPDEF: treat any other value as 64k */
+               if      (sz < SZ_64K)   sz = SZ_64K;
+               else if (sz < SZ_512M)  sz = SZ_512M;
+               else                    sz = 0;
+               break;
+       }
+
+       if (sz == 0)
+               return 0;
+
+       tmp &= ~(sz - 1);
+       if (kvm_pgtable_get_leaf(mmu->pgt, tmp, &pte, NULL))
+               goto again;
+       if (!(pte & PTE_VALID))
+               goto again;
+       level = FIELD_GET(KVM_NV_GUEST_MAP_SZ, pte);
+       if (!level)
+               goto again;
+
+       ttl |= level;
+
+       /*
+        * We now have found some level information in the shadow S2. Check
+        * that the resulting range is actually including the original IPA.
+        */
+       sz = ttl_to_size(ttl);
+       if (addr < (tmp + sz))
+               return ttl;
+
+       return 0;
+}
+
 unsigned long compute_tlb_inval_range(struct kvm_s2_mmu *mmu, u64 val)
 {
+       struct kvm *kvm = kvm_s2_mmu_to_kvm(mmu);
        unsigned long max_size;
        u8 ttl;
 
-       ttl = FIELD_GET(GENMASK_ULL(47, 44), val);
+       ttl = FIELD_GET(TLBI_TTL_MASK, val);
+
+       if (!ttl || !kvm_has_feat(kvm, ID_AA64MMFR2_EL1, TTL, IMP)) {
+               /* No TTL, check the shadow S2 for a hint */
+               u64 addr = (val & GENMASK_ULL(35, 0)) << 12;
+               ttl = get_guest_mapping_ttl(mmu, addr);
+       }
 
        max_size = ttl_to_size(ttl);