x86/msr: Move rdtsc{,_ordered}() to <asm/tsc.h>

Relocate rdtsc{,_ordered}() from <asm/msr.h> to <asm/tsc.h>.

[ mingo: Do not remove the <asm/tsc.h> inclusion from <asm/msr.h>
         just yet, to reduce -next breakages. We can do this later
	 on, separately, shortly before the next -rc1. ]

Signed-off-by: Xin Li (Intel) <xin@zytor.com>
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Acked-by: Dave Hansen <dave.hansen@linux.intel.com>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Brian Gerst <brgerst@gmail.com>
Cc: Juergen Gross <jgross@suse.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Kees Cook <keescook@chromium.org>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Josh Poimboeuf <jpoimboe@redhat.com>
Cc: Uros Bizjak <ubizjak@gmail.com>
Link: https://lore.kernel.org/r/20250427092027.1598740-3-xin@zytor.com

diff --git a/arch/x86/include/asm/msr.h b/arch/x86/include/asm/msr.h
index 35a78d2721a396cf8e24754845327c81406a467d..f5c0969fa3bbdab47372021528d58d1701d8a167 100644 (file)
--- a/arch/x86/include/asm/msr.h
+++ b/arch/x86/include/asm/msr.h
@@ -153,60 +153,6 @@ native_write_msr_safe(u32 msr, u32 low, u32 high)
 extern int rdmsr_safe_regs(u32 regs[8]);
 extern int wrmsr_safe_regs(u32 regs[8]);
 
-/**
- * rdtsc() - returns the current TSC without ordering constraints
- *
- * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
- * only ordering constraint it supplies is the ordering implied by
- * "asm volatile": it will put the RDTSC in the place you expect.  The
- * CPU can and will speculatively execute that RDTSC, though, so the
- * results can be non-monotonic if compared on different CPUs.
- */
-static __always_inline u64 rdtsc(void)
-{
-       EAX_EDX_DECLARE_ARGS(val, low, high);
-
-       asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
-
-       return EAX_EDX_VAL(val, low, high);
-}
-
-/**
- * rdtsc_ordered() - read the current TSC in program order
- *
- * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
- * It is ordered like a load to a global in-memory counter.  It should
- * be impossible to observe non-monotonic rdtsc_unordered() behavior
- * across multiple CPUs as long as the TSC is synced.
- */
-static __always_inline u64 rdtsc_ordered(void)
-{
-       EAX_EDX_DECLARE_ARGS(val, low, high);
-
-       /*
-        * The RDTSC instruction is not ordered relative to memory
-        * access.  The Intel SDM and the AMD APM are both vague on this
-        * point, but empirically an RDTSC instruction can be
-        * speculatively executed before prior loads.  An RDTSC
-        * immediately after an appropriate barrier appears to be
-        * ordered as a normal load, that is, it provides the same
-        * ordering guarantees as reading from a global memory location
-        * that some other imaginary CPU is updating continuously with a
-        * time stamp.
-        *
-        * Thus, use the preferred barrier on the respective CPU, aiming for
-        * RDTSCP as the default.
-        */
-       asm volatile(ALTERNATIVE_2("rdtsc",
-                                  "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
-                                  "rdtscp", X86_FEATURE_RDTSCP)
-                       : EAX_EDX_RET(val, low, high)
-                       /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
-                       :: "ecx");
-
-       return EAX_EDX_VAL(val, low, high);
-}
-
 static inline u64 native_read_pmc(int counter)
 {
        EAX_EDX_DECLARE_ARGS(val, low, high);

diff --git a/arch/x86/include/asm/tsc.h b/arch/x86/include/asm/tsc.h
index 94408a784c8e7e16ee66c22e04e68261f2feb716..4f7f09f505521db6e353491377bf6d2ba62230ef 100644 (file)
--- a/arch/x86/include/asm/tsc.h
+++ b/arch/x86/include/asm/tsc.h
@@ -5,10 +5,65 @@
 #ifndef _ASM_X86_TSC_H
 #define _ASM_X86_TSC_H
 
+#include <asm/asm.h>
 #include <asm/cpufeature.h>
 #include <asm/processor.h>
 #include <asm/msr.h>
 
+/**
+ * rdtsc() - returns the current TSC without ordering constraints
+ *
+ * rdtsc() returns the result of RDTSC as a 64-bit integer.  The
+ * only ordering constraint it supplies is the ordering implied by
+ * "asm volatile": it will put the RDTSC in the place you expect.  The
+ * CPU can and will speculatively execute that RDTSC, though, so the
+ * results can be non-monotonic if compared on different CPUs.
+ */
+static __always_inline u64 rdtsc(void)
+{
+       EAX_EDX_DECLARE_ARGS(val, low, high);
+
+       asm volatile("rdtsc" : EAX_EDX_RET(val, low, high));
+
+       return EAX_EDX_VAL(val, low, high);
+}
+
+/**
+ * rdtsc_ordered() - read the current TSC in program order
+ *
+ * rdtsc_ordered() returns the result of RDTSC as a 64-bit integer.
+ * It is ordered like a load to a global in-memory counter.  It should
+ * be impossible to observe non-monotonic rdtsc_unordered() behavior
+ * across multiple CPUs as long as the TSC is synced.
+ */
+static __always_inline u64 rdtsc_ordered(void)
+{
+       EAX_EDX_DECLARE_ARGS(val, low, high);
+
+       /*
+        * The RDTSC instruction is not ordered relative to memory
+        * access.  The Intel SDM and the AMD APM are both vague on this
+        * point, but empirically an RDTSC instruction can be
+        * speculatively executed before prior loads.  An RDTSC
+        * immediately after an appropriate barrier appears to be
+        * ordered as a normal load, that is, it provides the same
+        * ordering guarantees as reading from a global memory location
+        * that some other imaginary CPU is updating continuously with a
+        * time stamp.
+        *
+        * Thus, use the preferred barrier on the respective CPU, aiming for
+        * RDTSCP as the default.
+        */
+       asm volatile(ALTERNATIVE_2("rdtsc",
+                                  "lfence; rdtsc", X86_FEATURE_LFENCE_RDTSC,
+                                  "rdtscp", X86_FEATURE_RDTSCP)
+                       : EAX_EDX_RET(val, low, high)
+                       /* RDTSCP clobbers ECX with MSR_TSC_AUX. */
+                       :: "ecx");
+
+       return EAX_EDX_VAL(val, low, high);
+}
+
 /*
  * Standard way to access the cycle counter.
  */