x86/hyperv: Clean up hv_do_hypercall()

What used to be a simple few instructions has turned into a giant mess
(for x86_64). Not only does it use static_branch wrong, it mixes it
with dynamic branches for no apparent reason.

Notably it uses static_branch through an out-of-line function call,
which completely defeats the purpose, since instead of a simple
JMP/NOP site, you get a CALL+RET+TEST+Jcc sequence in return, which is
absolutely idiotic.

Add to that a dynamic test of hyperv_paravisor_present, something
which is set once and never changed.

Replace all this idiocy with a single direct function call to the
right hypercall variant.

Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Reviewed-by: Michael Kelley <mhklinux@outlook.com>
Tested-by: Michael Kelley <mhklinux@outlook.com>
Acked-by: Sean Christopherson <seanjc@google.com>
Link: https://lkml.kernel.org/r/20250714103440.897136093@infradead.org

diff --git a/arch/x86/hyperv/hv_init.c b/arch/x86/hyperv/hv_init.c
index afdbda2dd7b7e213289a272adcafed5e6e1e879f..5ef1e647ebc0adf4ef88d72dcebf47e17b5a3bd8 100644 (file)
--- a/arch/x86/hyperv/hv_init.c
+++ b/arch/x86/hyperv/hv_init.c
@@ -37,7 +37,27 @@
 #include <linux/export.h>
 
 void *hv_hypercall_pg;
+
+#ifdef CONFIG_X86_64
+u64 hv_std_hypercall(u64 control, u64 param1, u64 param2)
+{
+       u64 hv_status;
+
+       if (!hv_hypercall_pg)
+               return U64_MAX;
+
+       register u64 __r8 asm("r8") = param2;
+       asm volatile (CALL_NOSPEC
+                     : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+                       "+c" (control), "+d" (param1), "+r" (__r8)
+                     : THUNK_TARGET(hv_hypercall_pg)
+                     : "cc", "memory", "r9", "r10", "r11");
+
+       return hv_status;
+}
+#else
 EXPORT_SYMBOL_GPL(hv_hypercall_pg);
+#endif
 
 union hv_ghcb * __percpu *hv_ghcb_pg;
 

diff --git a/arch/x86/hyperv/ivm.c b/arch/x86/hyperv/ivm.c
index ade6c665c97e9cd9a68cab0287dbbe8b6c417fd2..21042fa4908829e4a56b152bd4016fc27ba0fc7d 100644 (file)
--- a/arch/x86/hyperv/ivm.c
+++ b/arch/x86/hyperv/ivm.c
@@ -385,9 +385,23 @@ int hv_snp_boot_ap(u32 apic_id, unsigned long start_ip, unsigned int cpu)
        return ret;
 }
 
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2)
+{
+       u64 hv_status;
+
+       register u64 __r8 asm("r8") = param2;
+       asm volatile("vmmcall"
+                    : "=a" (hv_status), ASM_CALL_CONSTRAINT,
+                      "+c" (control), "+d" (param1), "+r" (__r8)
+                    : : "cc", "memory", "r9", "r10", "r11");
+
+       return hv_status;
+}
+
 #else
 static inline void hv_ghcb_msr_write(u64 msr, u64 value) {}
 static inline void hv_ghcb_msr_read(u64 msr, u64 *value) {}
+u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_AMD_MEM_ENCRYPT */
 
 #ifdef CONFIG_INTEL_TDX_GUEST
@@ -437,6 +451,7 @@ u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2)
 #else
 static inline void hv_tdx_msr_write(u64 msr, u64 value) {}
 static inline void hv_tdx_msr_read(u64 msr, u64 *value) {}
+u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2) { return U64_MAX; }
 #endif /* CONFIG_INTEL_TDX_GUEST */
 
 #if defined(CONFIG_AMD_MEM_ENCRYPT) || defined(CONFIG_INTEL_TDX_GUEST)

diff --git a/arch/x86/include/asm/mshyperv.h b/arch/x86/include/asm/mshyperv.h
index abc4659f5809f9383a3475e658f4a77ad500a62d..605abd02158dd640fc4f33762c9f42389176f95d 100644 (file)
--- a/arch/x86/include/asm/mshyperv.h
+++ b/arch/x86/include/asm/mshyperv.h
@@ -6,6 +6,7 @@
 #include <linux/nmi.h>
 #include <linux/msi.h>
 #include <linux/io.h>
+#include <linux/static_call.h>
 #include <asm/nospec-branch.h>
 #include <asm/paravirt.h>
 #include <asm/msr.h>
@@ -39,16 +40,21 @@ static inline unsigned char hv_get_nmi_reason(void)
        return 0;
 }
 
-#if IS_ENABLED(CONFIG_HYPERV)
-extern bool hyperv_paravisor_present;
+extern u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
+extern u64 hv_snp_hypercall(u64 control, u64 param1, u64 param2);
+extern u64 hv_std_hypercall(u64 control, u64 param1, u64 param2);
 
+#if IS_ENABLED(CONFIG_HYPERV)
 extern void *hv_hypercall_pg;
 
 extern union hv_ghcb * __percpu *hv_ghcb_pg;
 
 bool hv_isolation_type_snp(void);
 bool hv_isolation_type_tdx(void);
-u64 hv_tdx_hypercall(u64 control, u64 param1, u64 param2);
+
+#ifdef CONFIG_X86_64
+DECLARE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
+#endif
 
 /*
  * DEFAULT INIT GPAT and SEGMENT LIMIT value in struct VMSA
@@ -65,37 +71,15 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
 {
        u64 input_address = input ? virt_to_phys(input) : 0;
        u64 output_address = output ? virt_to_phys(output) : 0;
-       u64 hv_status;
 
 #ifdef CONFIG_X86_64
-       if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-               return hv_tdx_hypercall(control, input_address, output_address);
-
-       if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-               __asm__ __volatile__("mov %[output_address], %%r8\n"
-                                    "vmmcall"
-                                    : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                                      "+c" (control), "+d" (input_address)
-                                    : [output_address] "r" (output_address)
-                                    : "cc", "memory", "r8", "r9", "r10", "r11");
-               return hv_status;
-       }
-
-       if (!hv_hypercall_pg)
-               return U64_MAX;
-
-       __asm__ __volatile__("mov %[output_address], %%r8\n"
-                            CALL_NOSPEC
-                            : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                              "+c" (control), "+d" (input_address)
-                            : [output_address] "r" (output_address),
-                              THUNK_TARGET(hv_hypercall_pg)
-                            : "cc", "memory", "r8", "r9", "r10", "r11");
+       return static_call_mod(hv_hypercall)(control, input_address, output_address);
 #else
        u32 input_address_hi = upper_32_bits(input_address);
        u32 input_address_lo = lower_32_bits(input_address);
        u32 output_address_hi = upper_32_bits(output_address);
        u32 output_address_lo = lower_32_bits(output_address);
+       u64 hv_status;
 
        if (!hv_hypercall_pg)
                return U64_MAX;
@@ -108,48 +92,30 @@ static inline u64 hv_do_hypercall(u64 control, void *input, void *output)
                               "D"(output_address_hi), "S"(output_address_lo),
                               THUNK_TARGET(hv_hypercall_pg)
                             : "cc", "memory");
-#endif /* !x86_64 */
        return hv_status;
+#endif /* !x86_64 */
 }
 
 /* Fast hypercall with 8 bytes of input and no output */
 static inline u64 _hv_do_fast_hypercall8(u64 control, u64 input1)
 {
-       u64 hv_status;
-
 #ifdef CONFIG_X86_64
-       if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-               return hv_tdx_hypercall(control, input1, 0);
-
-       if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-               __asm__ __volatile__(
-                               "vmmcall"
-                               : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                               "+c" (control), "+d" (input1)
-                               :: "cc", "r8", "r9", "r10", "r11");
-       } else {
-               __asm__ __volatile__(CALL_NOSPEC
-                                    : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                                      "+c" (control), "+d" (input1)
-                                    : THUNK_TARGET(hv_hypercall_pg)
-                                    : "cc", "r8", "r9", "r10", "r11");
-       }
+       return static_call_mod(hv_hypercall)(control, input1, 0);
 #else
-       {
-               u32 input1_hi = upper_32_bits(input1);
-               u32 input1_lo = lower_32_bits(input1);
-
-               __asm__ __volatile__ (CALL_NOSPEC
-                                     : "=A"(hv_status),
-                                       "+c"(input1_lo),
-                                       ASM_CALL_CONSTRAINT
-                                     : "A" (control),
-                                       "b" (input1_hi),
-                                       THUNK_TARGET(hv_hypercall_pg)
-                                     : "cc", "edi", "esi");
-       }
-#endif
+       u32 input1_hi = upper_32_bits(input1);
+       u32 input1_lo = lower_32_bits(input1);
+       u64 hv_status;
+
+       __asm__ __volatile__ (CALL_NOSPEC
+                             : "=A"(hv_status),
+                             "+c"(input1_lo),
+                             ASM_CALL_CONSTRAINT
+                             : "A" (control),
+                             "b" (input1_hi),
+                             THUNK_TARGET(hv_hypercall_pg)
+                             : "cc", "edi", "esi");
        return hv_status;
+#endif
 }
 
 static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
@@ -162,45 +128,24 @@ static inline u64 hv_do_fast_hypercall8(u16 code, u64 input1)
 /* Fast hypercall with 16 bytes of input */
 static inline u64 _hv_do_fast_hypercall16(u64 control, u64 input1, u64 input2)
 {
-       u64 hv_status;
-
 #ifdef CONFIG_X86_64
-       if (hv_isolation_type_tdx() && !hyperv_paravisor_present)
-               return hv_tdx_hypercall(control, input1, input2);
-
-       if (hv_isolation_type_snp() && !hyperv_paravisor_present) {
-               __asm__ __volatile__("mov %[input2], %%r8\n"
-                                    "vmmcall"
-                                    : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                                      "+c" (control), "+d" (input1)
-                                    : [input2] "r" (input2)
-                                    : "cc", "r8", "r9", "r10", "r11");
-       } else {
-               __asm__ __volatile__("mov %[input2], %%r8\n"
-                                    CALL_NOSPEC
-                                    : "=a" (hv_status), ASM_CALL_CONSTRAINT,
-                                      "+c" (control), "+d" (input1)
-                                    : [input2] "r" (input2),
-                                      THUNK_TARGET(hv_hypercall_pg)
-                                    : "cc", "r8", "r9", "r10", "r11");
-       }
+       return static_call_mod(hv_hypercall)(control, input1, input2);
 #else
-       {
-               u32 input1_hi = upper_32_bits(input1);
-               u32 input1_lo = lower_32_bits(input1);
-               u32 input2_hi = upper_32_bits(input2);
-               u32 input2_lo = lower_32_bits(input2);
-
-               __asm__ __volatile__ (CALL_NOSPEC
-                                     : "=A"(hv_status),
-                                       "+c"(input1_lo), ASM_CALL_CONSTRAINT
-                                     : "A" (control), "b" (input1_hi),
-                                       "D"(input2_hi), "S"(input2_lo),
-                                       THUNK_TARGET(hv_hypercall_pg)
-                                     : "cc");
-       }
-#endif
+       u32 input1_hi = upper_32_bits(input1);
+       u32 input1_lo = lower_32_bits(input1);
+       u32 input2_hi = upper_32_bits(input2);
+       u32 input2_lo = lower_32_bits(input2);
+       u64 hv_status;
+
+       __asm__ __volatile__ (CALL_NOSPEC
+                             : "=A"(hv_status),
+                             "+c"(input1_lo), ASM_CALL_CONSTRAINT
+                             : "A" (control), "b" (input1_hi),
+                             "D"(input2_hi), "S"(input2_lo),
+                             THUNK_TARGET(hv_hypercall_pg)
+                             : "cc");
        return hv_status;
+#endif
 }
 
 static inline u64 hv_do_fast_hypercall16(u16 code, u64 input1, u64 input2)

diff --git a/arch/x86/kernel/cpu/mshyperv.c b/arch/x86/kernel/cpu/mshyperv.c
index c78f860419d697c91349b28853c288fb6b2e41f6..3c71effe9ff56c13505b93457bd845984a19b223 100644 (file)
--- a/arch/x86/kernel/cpu/mshyperv.c
+++ b/arch/x86/kernel/cpu/mshyperv.c
@@ -38,10 +38,6 @@
 bool hv_nested;
 struct ms_hyperv_info ms_hyperv;
 
-/* Used in modules via hv_do_hypercall(): see arch/x86/include/asm/mshyperv.h */
-bool hyperv_paravisor_present __ro_after_init;
-EXPORT_SYMBOL_GPL(hyperv_paravisor_present);
-
 #if IS_ENABLED(CONFIG_HYPERV)
 static inline unsigned int hv_get_nested_msr(unsigned int reg)
 {
@@ -288,8 +284,18 @@ static void __init x86_setup_ops_for_tsc_pg_clock(void)
        old_restore_sched_clock_state = x86_platform.restore_sched_clock_state;
        x86_platform.restore_sched_clock_state = hv_restore_sched_clock_state;
 }
+
+#ifdef CONFIG_X86_64
+DEFINE_STATIC_CALL(hv_hypercall, hv_std_hypercall);
+EXPORT_STATIC_CALL_TRAMP_GPL(hv_hypercall);
+#define hypercall_update(hc) static_call_update(hv_hypercall, hc)
+#endif
 #endif /* CONFIG_HYPERV */
 
+#ifndef hypercall_update
+#define hypercall_update(hc) (void)hc
+#endif
+
 static uint32_t  __init ms_hyperv_platform(void)
 {
        u32 eax;
@@ -484,14 +490,14 @@ static void __init ms_hyperv_init_platform(void)
                        ms_hyperv.shared_gpa_boundary =
                                BIT_ULL(ms_hyperv.shared_gpa_boundary_bits);
 
-               hyperv_paravisor_present = !!ms_hyperv.paravisor_present;
-
                pr_info("Hyper-V: Isolation Config: Group A 0x%x, Group B 0x%x\n",
                        ms_hyperv.isolation_config_a, ms_hyperv.isolation_config_b);
 
 
                if (hv_get_isolation_type() == HV_ISOLATION_TYPE_SNP) {
                        static_branch_enable(&isolation_type_snp);
+                       if (!ms_hyperv.paravisor_present)
+                               hypercall_update(hv_snp_hypercall);
                } else if (hv_get_isolation_type() == HV_ISOLATION_TYPE_TDX) {
                        static_branch_enable(&isolation_type_tdx);
 
@@ -499,6 +505,7 @@ static void __init ms_hyperv_init_platform(void)
                        ms_hyperv.hints &= ~HV_X64_APIC_ACCESS_RECOMMENDED;
 
                        if (!ms_hyperv.paravisor_present) {
+                               hypercall_update(hv_tdx_hypercall);
                                /*
                                 * Mark the Hyper-V TSC page feature as disabled
                                 * in a TDX VM without paravisor so that the