KVM: MMU: Optimize is_last_gpte()

Instead of branchy code depending on level, gpte.ps, and mmu configuration,
prepare everything in a bitmap during mode changes and look it up during
runtime.

Reviewed-by: Xiao Guangrong <xiaoguangrong@linux.vnet.ibm.com>
Signed-off-by: Avi Kivity <avi@redhat.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 3318bde206a56958a117e59acbd44397c160d706..43aeb9422839fd040ae7fde4641d274c25593f54 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -298,6 +298,13 @@ struct kvm_mmu {
        u64 *lm_root;
        u64 rsvd_bits_mask[2][4];
 
+       /*
+        * Bitmap: bit set = last pte in walk
+        * index[0:1]: level (zero-based)
+        * index[2]: pte.ps
+        */
+       u8 last_pte_bitmap;
+
        bool nx;
 
        u64 pdptrs[4]; /* pae */

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 9c6188931f87d9edc44efd37db191aa974f2260e..d289fee1ffb8631c0b93f663104965fe7def7b11 100644 (file)
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -3447,6 +3447,15 @@ static inline unsigned gpte_access(struct kvm_vcpu *vcpu, u64 gpte)
        return access;
 }
 
+static inline bool is_last_gpte(struct kvm_mmu *mmu, unsigned level, unsigned gpte)
+{
+       unsigned index;
+
+       index = level - 1;
+       index |= (gpte & PT_PAGE_SIZE_MASK) >> (PT_PAGE_SIZE_SHIFT - 2);
+       return mmu->last_pte_bitmap & (1 << index);
+}
+
 #define PTTYPE 64
 #include "paging_tmpl.h"
 #undef PTTYPE
@@ -3548,6 +3557,24 @@ static void update_permission_bitmask(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu
        }
 }
 
+static void update_last_pte_bitmap(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu)
+{
+       u8 map;
+       unsigned level, root_level = mmu->root_level;
+       const unsigned ps_set_index = 1 << 2;  /* bit 2 of index: ps */
+
+       if (root_level == PT32E_ROOT_LEVEL)
+               --root_level;
+       /* PT_PAGE_TABLE_LEVEL always terminates */
+       map = 1 | (1 << ps_set_index);
+       for (level = PT_DIRECTORY_LEVEL; level <= root_level; ++level) {
+               if (level <= PT_PDPE_LEVEL
+                   && (mmu->root_level >= PT32E_ROOT_LEVEL || is_pse(vcpu)))
+                       map |= 1 << (ps_set_index | (level - 1));
+       }
+       mmu->last_pte_bitmap = map;
+}
+
 static int paging64_init_context_common(struct kvm_vcpu *vcpu,
                                        struct kvm_mmu *context,
                                        int level)
@@ -3557,6 +3584,7 @@ static int paging64_init_context_common(struct kvm_vcpu *vcpu,
 
        reset_rsvds_bits_mask(vcpu, context);
        update_permission_bitmask(vcpu, context);
+       update_last_pte_bitmap(vcpu, context);
 
        ASSERT(is_pae(vcpu));
        context->new_cr3 = paging_new_cr3;
@@ -3586,6 +3614,7 @@ static int paging32_init_context(struct kvm_vcpu *vcpu,
 
        reset_rsvds_bits_mask(vcpu, context);
        update_permission_bitmask(vcpu, context);
+       update_last_pte_bitmap(vcpu, context);
 
        context->new_cr3 = paging_new_cr3;
        context->page_fault = paging32_page_fault;
@@ -3647,6 +3676,7 @@ static int init_kvm_tdp_mmu(struct kvm_vcpu *vcpu)
        }
 
        update_permission_bitmask(vcpu, context);
+       update_last_pte_bitmap(vcpu, context);
 
        return 0;
 }
@@ -3724,6 +3754,7 @@ static int init_kvm_nested_mmu(struct kvm_vcpu *vcpu)
        }
 
        update_permission_bitmask(vcpu, g_context);
+       update_last_pte_bitmap(vcpu, g_context);
 
        return 0;
 }

diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index 584660775d08170b2ef903c13a4e771d125ca7c7..69871080e8663c76fed7b94783f9b6869e552ac8 100644 (file)
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -20,7 +20,8 @@
 #define PT_ACCESSED_MASK (1ULL << PT_ACCESSED_SHIFT)
 #define PT_DIRTY_SHIFT 6
 #define PT_DIRTY_MASK (1ULL << PT_DIRTY_SHIFT)
-#define PT_PAGE_SIZE_MASK (1ULL << 7)
+#define PT_PAGE_SIZE_SHIFT 7
+#define PT_PAGE_SIZE_MASK (1ULL << PT_PAGE_SIZE_SHIFT)
 #define PT_PAT_MASK (1ULL << 7)
 #define PT_GLOBAL_MASK (1ULL << 8)
 #define PT64_NX_SHIFT 63

diff --git a/arch/x86/kvm/paging_tmpl.h b/arch/x86/kvm/paging_tmpl.h
index 1b4c14d235a0ed027a205b4d499eea0bc94d94d9..134ea7b1c585a2876469c610cc661ed4be27b063 100644 (file)
--- a/arch/x86/kvm/paging_tmpl.h
+++ b/arch/x86/kvm/paging_tmpl.h
@@ -103,24 +103,6 @@ static int FNAME(cmpxchg_gpte)(struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
        return (ret != orig_pte);
 }
 
-static bool FNAME(is_last_gpte)(struct guest_walker *walker,
-                               struct kvm_vcpu *vcpu, struct kvm_mmu *mmu,
-                               pt_element_t gpte)
-{
-       if (walker->level == PT_PAGE_TABLE_LEVEL)
-               return true;
-
-       if ((walker->level == PT_DIRECTORY_LEVEL) && is_large_pte(gpte) &&
-           (PTTYPE == 64 || is_pse(vcpu)))
-               return true;
-
-       if ((walker->level == PT_PDPE_LEVEL) && is_large_pte(gpte) &&
-           (mmu->root_level == PT64_ROOT_LEVEL))
-               return true;
-
-       return false;
-}
-
 static int FNAME(update_accessed_dirty_bits)(struct kvm_vcpu *vcpu,
                                             struct kvm_mmu *mmu,
                                             struct guest_walker *walker,
@@ -247,7 +229,7 @@ retry_walk:
                pte_access = pt_access & gpte_access(vcpu, pte);
 
                walker->ptes[walker->level - 1] = pte;
-       } while (!FNAME(is_last_gpte)(walker, vcpu, mmu, pte));
+       } while (!is_last_gpte(mmu, walker->level, pte));
 
        eperm |= permission_fault(mmu, pte_access, access);
        if (unlikely(eperm)) {