KVM: nVMX: Skip IBPB when switching between vmcs01 and vmcs02

commit 5c911beff20aa8639e7a1f28988736c13e03ed54 upstream.

Skip the Indirect Branch Prediction Barrier that is triggered on a VMCS
switch when running with spectre_v2_user=on/auto if the switch is
between two VMCSes in the same guest, i.e. between vmcs01 and vmcs02.
The IBPB is intended to prevent one guest from attacking another, which
is unnecessary in the nested case as it's the same guest from KVM's
perspective.

This all but eliminates the overhead observed for nested VMX transitions
when running with CONFIG_RETPOLINE=y and spectre_v2_user=on/auto, which
can be significant, e.g. roughly 3x on current systems.

Reported-by: Alexander Graf <graf@amazon.com>
Cc: KarimAllah Raslan <karahmed@amazon.de>
Cc: stable@vger.kernel.org
Fixes: 15d45071523d ("KVM/x86: Add IBPB support")
Signed-off-by: Sean Christopherson <sean.j.christopherson@intel.com>
Message-Id: <20200501163117.4655-1-sean.j.christopherson@intel.com>
[Invert direction of bool argument. - Paolo]
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
Signed-off-by: Greg Kroah-Hartman <gregkh@linuxfoundation.org>

diff --git a/arch/x86/kvm/vmx/nested.c b/arch/x86/kvm/vmx/nested.c
index e44f33c82332505b057ae016cf0d5a5c0cba23d0..bf1c0e888673a9d07d2ad1fa5104673243b2bdde 100644 (file)
--- a/arch/x86/kvm/vmx/nested.c
+++ b/arch/x86/kvm/vmx/nested.c
@@ -303,7 +303,7 @@ static void vmx_switch_vmcs(struct kvm_vcpu *vcpu, struct loaded_vmcs *vmcs)
        cpu = get_cpu();
        prev = vmx->loaded_vmcs;
        vmx->loaded_vmcs = vmcs;
-       vmx_vcpu_load_vmcs(vcpu, cpu);
+       vmx_vcpu_load_vmcs(vcpu, cpu, prev);
        vmx_sync_vmcs_host_state(vmx, prev);
        put_cpu();
 

diff --git a/arch/x86/kvm/vmx/vmx.c b/arch/x86/kvm/vmx/vmx.c
index 9b63ac8d97eeb624ff14c6d16f61bf2275dd5200..d7aa0dfab8bbd8924b0a60c2199621b60d62b12e 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.c
+++ b/arch/x86/kvm/vmx/vmx.c
@@ -1306,10 +1306,12 @@ after_clear_sn:
                pi_set_on(pi_desc);
 }
 
-void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu)
+void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
+                       struct loaded_vmcs *buddy)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
        bool already_loaded = vmx->loaded_vmcs->cpu == cpu;
+       struct vmcs *prev;
 
        if (!already_loaded) {
                loaded_vmcs_clear(vmx->loaded_vmcs);
@@ -1328,10 +1330,18 @@ void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu)
                local_irq_enable();
        }
 
-       if (per_cpu(current_vmcs, cpu) != vmx->loaded_vmcs->vmcs) {
+       prev = per_cpu(current_vmcs, cpu);
+       if (prev != vmx->loaded_vmcs->vmcs) {
                per_cpu(current_vmcs, cpu) = vmx->loaded_vmcs->vmcs;
                vmcs_load(vmx->loaded_vmcs->vmcs);
-               indirect_branch_prediction_barrier();
+
+               /*
+                * No indirect branch prediction barrier needed when switching
+                * the active VMCS within a guest, e.g. on nested VM-Enter.
+                * The L1 VMM can protect itself with retpolines, IBPB or IBRS.
+                */
+               if (!buddy || WARN_ON_ONCE(buddy->vmcs != prev))
+                       indirect_branch_prediction_barrier();
        }
 
        if (!already_loaded) {
@@ -1368,7 +1378,7 @@ void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
 {
        struct vcpu_vmx *vmx = to_vmx(vcpu);
 
-       vmx_vcpu_load_vmcs(vcpu, cpu);
+       vmx_vcpu_load_vmcs(vcpu, cpu, NULL);
 
        vmx_vcpu_pi_load(vcpu, cpu);
 

diff --git a/arch/x86/kvm/vmx/vmx.h b/arch/x86/kvm/vmx/vmx.h
index aab9df55336ef6e46e102450365f7ef28457e823..6bd7e552c5346683c0d198c8a7184dbae9180f89 100644 (file)
--- a/arch/x86/kvm/vmx/vmx.h
+++ b/arch/x86/kvm/vmx/vmx.h
@@ -317,7 +317,8 @@ struct kvm_vmx {
 };
 
 bool nested_vmx_allowed(struct kvm_vcpu *vcpu);
-void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu);
+void vmx_vcpu_load_vmcs(struct kvm_vcpu *vcpu, int cpu,
+                       struct loaded_vmcs *buddy);
 void vmx_vcpu_load(struct kvm_vcpu *vcpu, int cpu);
 int allocate_vpid(void);
 void free_vpid(int vpid);