kvm: add halt_poll_ns module parameter

This patch introduces a new module parameter for the KVM module; when it
is present, KVM attempts a bit of polling on every HLT before scheduling
itself out via kvm_vcpu_block.

This parameter helps a lot for latency-bound workloads---in particular
I tested it with O_DSYNC writes with a battery-backed disk in the host.
In this case, writes are fast (because the data doesn't have to go all
the way to the platters) but they cannot be merged by either the host or
the guest.  KVM's performance here is usually around 30% of bare metal,
or 50% if you use cache=directsync or cache=writethrough (these
parameters avoid that the guest sends pointless flush requests, and
at the same time they are not slow because of the battery-backed cache).
The bad performance happens because on every halt the host CPU decides
to halt itself too.  When the interrupt comes, the vCPU thread is then
migrated to a new physical CPU, and in general the latency is horrible
because the vCPU thread has to be scheduled back in.

With this patch performance reaches 60-65% of bare metal and, more
important, 99% of what you get if you use idle=poll in the guest.  This
means that the tunable gets rid of this particular bottleneck, and more
work can be done to improve performance in the kernel or QEMU.

Of course there is some price to pay; every time an otherwise idle vCPUs
is interrupted by an interrupt, it will poll unnecessarily and thus
impose a little load on the host.  The above results were obtained with
a mostly random value of the parameter (500000), and the load was around
1.5-2.5% CPU usage on one of the host's core for each idle guest vCPU.

The patch also adds a new stat, /sys/kernel/debug/kvm/halt_successful_poll,
that can be used to tune the parameter.  It counts how many HLT
instructions received an interrupt during the polling period; each
successful poll avoids that Linux schedules the VCPU thread out and back
in, and may also avoid a likely trip to C1 and back for the physical CPU.

While the VM is idle, a Linux 4 VCPU VM halts around 10 times per second.
Of these halts, almost all are failed polls.  During the benchmark,
instead, basically all halts end within the polling period, except a more
or less constant stream of 50 per second coming from vCPUs that are not
running the benchmark.  The wasted time is thus very low.  Things may
be slightly different for Windows VMs, which have a ~10 ms timer tick.

The effect is also visible on Marcelo's recently-introduced latency
test for the TSC deadline timer.  Though of course a non-RT kernel has
awful latency bounds, the latency of the timer is around 8000-10000 clock
cycles compared to 20000-120000 without setting halt_poll_ns.  For the TSC
deadline timer, thus, the effect is both a smaller average latency and
a smaller variance.

Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

diff --git a/arch/arm/include/asm/kvm_host.h b/arch/arm/include/asm/kvm_host.h
index bde494654bccb6a7434ed81f355fbdd426a43b97..6a79314bc1dfcf8055dfb4b790d0d35a2e2bec39 100644 (file)
--- a/arch/arm/include/asm/kvm_host.h
+++ b/arch/arm/include/asm/kvm_host.h
@@ -148,6 +148,7 @@ struct kvm_vm_stat {
 };
 
 struct kvm_vcpu_stat {
+       u32 halt_successful_poll;
        u32 halt_wakeup;
 };
 

diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
index 2c49aa4ac818596540c97694667fb076cbe7c363..8efde89613f26246711a80f5a4f86aa221797a0b 100644 (file)
--- a/arch/arm64/include/asm/kvm_host.h
+++ b/arch/arm64/include/asm/kvm_host.h
@@ -165,6 +165,7 @@ struct kvm_vm_stat {
 };
 
 struct kvm_vcpu_stat {
+       u32 halt_successful_poll;
        u32 halt_wakeup;
 };
 

diff --git a/arch/mips/include/asm/kvm_host.h b/arch/mips/include/asm/kvm_host.h
index f2c249796ea8baa4cb6cc96f79c7efeaa35400d2..ac4fc716062b791003c76f5572d56863bcdcb2cd 100644 (file)
--- a/arch/mips/include/asm/kvm_host.h
+++ b/arch/mips/include/asm/kvm_host.h
@@ -120,6 +120,7 @@ struct kvm_vcpu_stat {
        u32 resvd_inst_exits;
        u32 break_inst_exits;
        u32 flush_dcache_exits;
+       u32 halt_successful_poll;
        u32 halt_wakeup;
 };
 

diff --git a/arch/mips/kvm/mips.c b/arch/mips/kvm/mips.c
index e97b907840311d2bffa258443e2d72483605ac61..c9eccf5df912037e2b71bbb4a7dddd2a1d2d866e 100644 (file)
--- a/arch/mips/kvm/mips.c
+++ b/arch/mips/kvm/mips.c
@@ -49,6 +49,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "resvd_inst",   VCPU_STAT(resvd_inst_exits),   KVM_STAT_VCPU },
        { "break_inst",   VCPU_STAT(break_inst_exits),   KVM_STAT_VCPU },
        { "flush_dcache", VCPU_STAT(flush_dcache_exits), KVM_STAT_VCPU },
+       { "halt_successful_poll", VCPU_STAT(halt_successful_poll), KVM_STAT_VCPU },
        { "halt_wakeup",  VCPU_STAT(halt_wakeup),        KVM_STAT_VCPU },
        {NULL}
 };

diff --git a/arch/powerpc/include/asm/kvm_host.h b/arch/powerpc/include/asm/kvm_host.h
index 7efd666a3fa7801f4a19e559541d7c7c7ebaf95c..8ef05121d3cd90b51e1215cdf75a60368e18970c 100644 (file)
--- a/arch/powerpc/include/asm/kvm_host.h
+++ b/arch/powerpc/include/asm/kvm_host.h
@@ -107,6 +107,7 @@ struct kvm_vcpu_stat {
        u32 emulated_inst_exits;
        u32 dec_exits;
        u32 ext_intr_exits;
+       u32 halt_successful_poll;
        u32 halt_wakeup;
        u32 dbell_exits;
        u32 gdbell_exits;

diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c
index 888bf466d8c6bb8e33088472c5573081afe684a6..cfbcdc65420195c566c9541a38d9fc8aed0b80d9 100644 (file)
--- a/arch/powerpc/kvm/book3s.c
+++ b/arch/powerpc/kvm/book3s.c
@@ -52,6 +52,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "dec",         VCPU_STAT(dec_exits) },
        { "ext_intr",    VCPU_STAT(ext_intr_exits) },
        { "queue_intr",  VCPU_STAT(queue_intr) },
+       { "halt_successful_poll", VCPU_STAT(halt_successful_poll), },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "pf_storage",  VCPU_STAT(pf_storage) },
        { "sp_storage",  VCPU_STAT(sp_storage) },

diff --git a/arch/powerpc/kvm/booke.c b/arch/powerpc/kvm/booke.c
index 9b55dec2d6cc11846576abe16f286c87dc873b50..6c1316a15a2729a67ed61bcb6e2ce8b90af6bdbe 100644 (file)
--- a/arch/powerpc/kvm/booke.c
+++ b/arch/powerpc/kvm/booke.c
@@ -62,6 +62,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "inst_emu",   VCPU_STAT(emulated_inst_exits) },
        { "dec",        VCPU_STAT(dec_exits) },
        { "ext_intr",   VCPU_STAT(ext_intr_exits) },
+       { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "doorbell", VCPU_STAT(dbell_exits) },
        { "guest doorbell", VCPU_STAT(gdbell_exits) },

diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index d1ecc7fd05790434f8402402feef7a3e1509cd3f..f79058e3fd98f1d5c9c5e6a5b0f45ec35ff0e146 100644 (file)
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -196,6 +196,7 @@ struct kvm_vcpu_stat {
        u32 exit_stop_request;
        u32 exit_validity;
        u32 exit_instruction;
+       u32 halt_successful_poll;
        u32 halt_wakeup;
        u32 instruction_lctl;
        u32 instruction_lctlg;

diff --git a/arch/s390/kvm/kvm-s390.c b/arch/s390/kvm/kvm-s390.c
index b2371c0fd1f8cbecf406599dea6966f8cedfc546..1dbab2340a6623732ad8bed18327cadb7408983d 100644 (file)
--- a/arch/s390/kvm/kvm-s390.c
+++ b/arch/s390/kvm/kvm-s390.c
@@ -51,6 +51,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "exit_instruction", VCPU_STAT(exit_instruction) },
        { "exit_program_interruption", VCPU_STAT(exit_program_interruption) },
        { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) },
+       { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "instruction_lctlg", VCPU_STAT(instruction_lctlg) },
        { "instruction_lctl", VCPU_STAT(instruction_lctl) },

diff --git a/arch/x86/include/asm/kvm_host.h b/arch/x86/include/asm/kvm_host.h
index 848947ac6ade478e453b3ef786159463761b0c75..a236e39cc385a4a468e745ce0d9fdb1e2d08341b 100644 (file)
--- a/arch/x86/include/asm/kvm_host.h
+++ b/arch/x86/include/asm/kvm_host.h
@@ -655,6 +655,7 @@ struct kvm_vcpu_stat {
        u32 irq_window_exits;
        u32 nmi_window_exits;
        u32 halt_exits;
+       u32 halt_successful_poll;
        u32 halt_wakeup;
        u32 request_irq_exits;
        u32 irq_exits;

diff --git a/arch/x86/kvm/x86.c b/arch/x86/kvm/x86.c
index 1373e04e1f19b18ddc06488c10a196473f6eff4b..bd7a70be41b35fa93ad952060fc2c7f432e5628e 100644 (file)
--- a/arch/x86/kvm/x86.c
+++ b/arch/x86/kvm/x86.c
@@ -145,6 +145,7 @@ struct kvm_stats_debugfs_item debugfs_entries[] = {
        { "irq_window", VCPU_STAT(irq_window_exits) },
        { "nmi_window", VCPU_STAT(nmi_window_exits) },
        { "halt_exits", VCPU_STAT(halt_exits) },
+       { "halt_successful_poll", VCPU_STAT(halt_successful_poll) },
        { "halt_wakeup", VCPU_STAT(halt_wakeup) },
        { "hypercalls", VCPU_STAT(hypercalls) },
        { "request_irq", VCPU_STAT(request_irq_exits) },

diff --git a/include/trace/events/kvm.h b/include/trace/events/kvm.h
index 6edf1f2028cdb0e0801af85183e02d0416fa73ed..6bfe7eec1c2c7884a933c3ab24541c817bd70fae 100644 (file)
--- a/include/trace/events/kvm.h
+++ b/include/trace/events/kvm.h
@@ -37,6 +37,25 @@ TRACE_EVENT(kvm_userspace_exit,
                  __entry->errno < 0 ? -__entry->errno : __entry->reason)
 );
 
+TRACE_EVENT(kvm_vcpu_wakeup,
+           TP_PROTO(__u64 ns, bool waited),
+           TP_ARGS(ns, waited),
+
+       TP_STRUCT__entry(
+               __field(        __u64,          ns              )
+               __field(        bool,           waited          )
+       ),
+
+       TP_fast_assign(
+               __entry->ns             = ns;
+               __entry->waited         = waited;
+       ),
+
+       TP_printk("%s time %lld ns",
+                 __entry->waited ? "wait" : "poll",
+                 __entry->ns)
+);
+
 #if defined(CONFIG_HAVE_KVM_IRQFD)
 TRACE_EVENT(kvm_set_irq,
        TP_PROTO(unsigned int gsi, int level, int irq_source_id),

diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 0c281760a1c5f771962afdb49c48fdfd0efc61ac..32449e0e9aa8a0d62a35d66e328940c19fa7936a 100644 (file)
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -66,6 +66,9 @@
 MODULE_AUTHOR("Qumranet");
 MODULE_LICENSE("GPL");
 
+unsigned int halt_poll_ns = 0;
+module_param(halt_poll_ns, uint, S_IRUGO | S_IWUSR);
+
 /*
  * Ordering of locks:
  *
@@ -1813,29 +1816,60 @@ void mark_page_dirty(struct kvm *kvm, gfn_t gfn)
 }
 EXPORT_SYMBOL_GPL(mark_page_dirty);
 
+static int kvm_vcpu_check_block(struct kvm_vcpu *vcpu)
+{
+       if (kvm_arch_vcpu_runnable(vcpu)) {
+               kvm_make_request(KVM_REQ_UNHALT, vcpu);
+               return -EINTR;
+       }
+       if (kvm_cpu_has_pending_timer(vcpu))
+               return -EINTR;
+       if (signal_pending(current))
+               return -EINTR;
+
+       return 0;
+}
+
 /*
  * The vCPU has executed a HLT instruction with in-kernel mode enabled.
  */
 void kvm_vcpu_block(struct kvm_vcpu *vcpu)
 {
+       ktime_t start, cur;
        DEFINE_WAIT(wait);
+       bool waited = false;
+
+       start = cur = ktime_get();
+       if (halt_poll_ns) {
+               ktime_t stop = ktime_add_ns(ktime_get(), halt_poll_ns);
+               do {
+                       /*
+                        * This sets KVM_REQ_UNHALT if an interrupt
+                        * arrives.
+                        */
+                       if (kvm_vcpu_check_block(vcpu) < 0) {
+                               ++vcpu->stat.halt_successful_poll;
+                               goto out;
+                       }
+                       cur = ktime_get();
+               } while (single_task_running() && ktime_before(cur, stop));
+       }
 
        for (;;) {
                prepare_to_wait(&vcpu->wq, &wait, TASK_INTERRUPTIBLE);
 
-               if (kvm_arch_vcpu_runnable(vcpu)) {
-                       kvm_make_request(KVM_REQ_UNHALT, vcpu);
-                       break;
-               }
-               if (kvm_cpu_has_pending_timer(vcpu))
-                       break;
-               if (signal_pending(current))
+               if (kvm_vcpu_check_block(vcpu) < 0)
                        break;
 
+               waited = true;
                schedule();
        }
 
        finish_wait(&vcpu->wq, &wait);
+       cur = ktime_get();
+
+out:
+       trace_kvm_vcpu_wakeup(ktime_to_ns(cur) - ktime_to_ns(start), waited);
 }
 EXPORT_SYMBOL_GPL(kvm_vcpu_block);