KVM: x86/mmu: Lazily allocate memslot rmaps

If the TDP MMU is in use, wait to allocate the rmaps until the shadow
MMU is actually used. (i.e. a nested VM is launched.) This saves memory
equal to 0.2% of guest memory in cases where the TDP MMU is used and
there are no nested guests involved.

Signed-off-by: Ben Gardon <bgardon@google.com>
Message-Id: <20210518173414.450044-8-bgardon@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 11798a9ff3e9df2c1336f485cbccb27491273734..dadb545c429f6a76456334def77cfb1e994abce7 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -1869,4 +1869,6 @@ static inline int kvm_cpu_get_apicid(int mps_cpu)
 
 int kvm_cpu_dirty_log_size(void);
 
+int alloc_all_memslots_rmaps(struct kvm *kvm);
+
 #endif /* _ASM_X86_KVM_HOST_H */

diff --git a/arch/x86/kvm/mmu.h b/arch/x86/kvm/mmu.h
index af09c47b1aa22bbe137525c1f15a3db3863e530b..9d8550af994c4f2493691a4a5d24a6415ecdcc7c 100644 (file)
--- a/arch/x86/kvm/mmu.h
+++ b/arch/x86/kvm/mmu.h
@@ -234,7 +234,12 @@ void kvm_mmu_pre_destroy_vm(struct kvm *kvm);
 
 static inline bool kvm_memslots_have_rmaps(struct kvm *kvm)
 {
-       return kvm->arch.memslots_have_rmaps;
+       /*
+        * Read memslot_have_rmaps before rmap pointers.  Hence, threads reading
+        * memslots_have_rmaps in any lock context are guaranteed to see the
+        * pointers.  Pairs with smp_store_release in alloc_all_memslots_rmaps.
+        */
+       return smp_load_acquire(&kvm->arch.memslots_have_rmaps);
 }
 
 #endif

diff --git a/arch/x86/kvm/mmu/mmu.c b/arch/x86/kvm/mmu/mmu.c
index 2131f71577bc62e7df38863fb75f238aa114dbcd..aa9e77f406d9cdc81e71b7c8c930ef1f1d78abeb 100644 (file)
--- a/arch/x86/kvm/mmu/mmu.c
+++ b/arch/x86/kvm/mmu/mmu.c
@@ -3312,6 +3312,10 @@ static int mmu_alloc_shadow_roots(struct kvm_vcpu *vcpu)
                }
        }
 
+       r = alloc_all_memslots_rmaps(vcpu->kvm);
+       if (r)
+               return r;
+
        write_lock(&vcpu->kvm->mmu_lock);
        r = make_mmu_pages_available(vcpu);
        if (r < 0)
@@ -5523,9 +5527,13 @@ void kvm_mmu_init_vm(struct kvm *kvm)
 {
        struct kvm_page_track_notifier_node *node = &kvm->arch.mmu_sp_tracker;
 
-       kvm_mmu_init_tdp_mmu(kvm);
-
-       kvm->arch.memslots_have_rmaps = true;
+       if (!kvm_mmu_init_tdp_mmu(kvm))
+               /*
+                * No smp_load/store wrappers needed here as we are in
+                * VM init and there cannot be any memslots / other threads
+                * accessing this struct kvm yet.
+                */
+               kvm->arch.memslots_have_rmaps = true;
 
        node->track_write = kvm_mmu_pte_write;
        node->track_flush_slot = kvm_mmu_invalidate_zap_pages_in_memslot;

diff --git a/arch/x86/kvm/mmu/tdp_mmu.c b/arch/x86/kvm/mmu/tdp_mmu.c
index 6b6dfcdcb17979c8e590293a2b0675edf4edcaeb..cc13e001f3de08ce450fda139cb7b2884ba111d1 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.c
+++ b/arch/x86/kvm/mmu/tdp_mmu.c
@@ -14,10 +14,10 @@ static bool __read_mostly tdp_mmu_enabled = false;
 module_param_named(tdp_mmu, tdp_mmu_enabled, bool, 0644);
 
 /* Initializes the TDP MMU for the VM, if enabled. */
-void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
+bool kvm_mmu_init_tdp_mmu(struct kvm *kvm)
 {
        if (!tdp_enabled || !READ_ONCE(tdp_mmu_enabled))
-               return;
+               return false;
 
        /* This should not be changed for the lifetime of the VM. */
        kvm->arch.tdp_mmu_enabled = true;
@@ -25,6 +25,8 @@ void kvm_mmu_init_tdp_mmu(struct kvm *kvm)
        INIT_LIST_HEAD(&kvm->arch.tdp_mmu_roots);
        spin_lock_init(&kvm->arch.tdp_mmu_pages_lock);
        INIT_LIST_HEAD(&kvm->arch.tdp_mmu_pages);
+
+       return true;
 }
 
 static __always_inline void kvm_lockdep_assert_mmu_lock_held(struct kvm *kvm,

diff --git a/arch/x86/kvm/mmu/tdp_mmu.h b/arch/x86/kvm/mmu/tdp_mmu.h
index a861570fcd7cd09332c51c1d2f2c9ed78a42dff1..f7a7990da11dad21b93d90ea3ad751f3bb225bc8 100644 (file)
--- a/arch/x86/kvm/mmu/tdp_mmu.h
+++ b/arch/x86/kvm/mmu/tdp_mmu.h
@@ -81,12 +81,12 @@ int kvm_tdp_mmu_get_walk(struct kvm_vcpu *vcpu, u64 addr, u64 *sptes,
                         int *root_level);
 
 #ifdef CONFIG_X86_64
-void kvm_mmu_init_tdp_mmu(struct kvm *kvm);
+bool kvm_mmu_init_tdp_mmu(struct kvm *kvm);
 void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm);
 static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return kvm->arch.tdp_mmu_enabled; }
 static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return sp->tdp_mmu_page; }
 #else
-static inline void kvm_mmu_init_tdp_mmu(struct kvm *kvm) {}
+static inline bool kvm_mmu_init_tdp_mmu(struct kvm *kvm) { return false; }
 static inline void kvm_mmu_uninit_tdp_mmu(struct kvm *kvm) {}
 static inline bool is_tdp_mmu_enabled(struct kvm *kvm) { return false; }
 static inline bool is_tdp_mmu_page(struct kvm_mmu_page *sp) { return false; }

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index ddeff81f90a4821908486ed16853f65d908a48a7..e838e999ab49de0d66d215c2348cab1c501c3f41 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -10952,6 +10952,8 @@ static int memslot_rmap_alloc(struct kvm_memory_slot *slot,
                int lpages = gfn_to_index(slot->base_gfn + npages - 1,
                                          slot->base_gfn, level) + 1;
 
+               WARN_ON(slot->arch.rmap[i]);
+
                slot->arch.rmap[i] = kvcalloc(lpages, sz, GFP_KERNEL_ACCOUNT);
                if (!slot->arch.rmap[i]) {
                        memslot_rmap_free(slot);
@@ -10962,6 +10964,50 @@ static int memslot_rmap_alloc(struct kvm_memory_slot *slot,
        return 0;
 }
 
+int alloc_all_memslots_rmaps(struct kvm *kvm)
+{
+       struct kvm_memslots *slots;
+       struct kvm_memory_slot *slot;
+       int r, i;
+
+       /*
+        * Check if memslots alreday have rmaps early before acquiring
+        * the slots_arch_lock below.
+        */
+       if (kvm_memslots_have_rmaps(kvm))
+               return 0;
+
+       mutex_lock(&kvm->slots_arch_lock);
+
+       /*
+        * Read memslots_have_rmaps again, under the slots arch lock,
+        * before allocating the rmaps
+        */
+       if (kvm_memslots_have_rmaps(kvm)) {
+               mutex_unlock(&kvm->slots_arch_lock);
+               return 0;
+       }
+
+       for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
+               slots = __kvm_memslots(kvm, i);
+               kvm_for_each_memslot(slot, slots) {
+                       r = memslot_rmap_alloc(slot, slot->npages);
+                       if (r) {
+                               mutex_unlock(&kvm->slots_arch_lock);
+                               return r;
+                       }
+               }
+       }
+
+       /*
+        * Ensure that memslots_have_rmaps becomes true strictly after
+        * all the rmap pointers are set.
+        */
+       smp_store_release(&kvm->arch.memslots_have_rmaps, true);
+       mutex_unlock(&kvm->slots_arch_lock);
+       return 0;
+}
+
 static int kvm_alloc_memslot_metadata(struct kvm *kvm,
                                      struct kvm_memory_slot *slot,
                                      unsigned long npages)