x86/smpboot: Remove initial_stack on 64-bit

Eliminating global variables from the CPU startup path in order to simplify
it and facilitate parallel startup.

Remove initial_stack, and load RSP from current_task->thread.sp instead.

Signed-off-by: Brian Gerst <brgerst@gmail.com>
Reviewed-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Tested-by: Usama Arif <usama.arif@bytedance.com>
Signed-off-by: Usama Arif <usama.arif@bytedance.com>

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index 4e35c66edeb7dc028cccb9737769704d18fb2096..bdde7316e75baaa2ad6df1f8bc4e33fbcecaa2b3 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -648,7 +648,11 @@ static inline void spin_lock_prefetch(const void *x)
 #define KSTK_ESP(task)         (task_pt_regs(task)->sp)
 
 #else
-#define INIT_THREAD { }
+extern unsigned long __end_init_task[];
+
+#define INIT_THREAD {                                                      \
+       .sp     = (unsigned long)&__end_init_task - sizeof(struct pt_regs), \
+}
 
 extern unsigned long KSTK_ESP(struct task_struct *task);
 

diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h
index b4dbb20dab1a11d509b4f702f4bde6a6269b4d18..bf2c51df9e0b39f6f171696e6bda246d911cb5ce 100644 (file)
--- a/arch/x86/include/asm/smp.h
+++ b/arch/x86/include/asm/smp.h
@@ -199,5 +199,8 @@ extern void nmi_selftest(void);
 #define nmi_selftest() do { } while (0)
 #endif
 
-#endif /* __ASSEMBLY__ */
+extern unsigned int smpboot_control;
+
+#endif /* !__ASSEMBLY__ */
+
 #endif /* _ASM_X86_SMP_H */

diff --git a/arch/x86/kernel/acpi/sleep.c b/arch/x86/kernel/acpi/sleep.c
index 3b7f4cdbf2e0a12b9bfcb2e485327d01f20cbf87..ab6e29b32c0454754843506949dd0bd3a53a38ef 100644 (file)
--- a/arch/x86/kernel/acpi/sleep.c
+++ b/arch/x86/kernel/acpi/sleep.c
@@ -111,13 +111,14 @@ int x86_acpi_suspend_lowlevel(void)
        saved_magic = 0x12345678;
 #else /* CONFIG_64BIT */
 #ifdef CONFIG_SMP
-       initial_stack = (unsigned long)temp_stack + sizeof(temp_stack);
+       current->thread.sp = (unsigned long)temp_stack + sizeof(temp_stack);
        early_gdt_descr.address =
                        (unsigned long)get_cpu_gdt_rw(smp_processor_id());
        initial_gs = per_cpu_offset(smp_processor_id());
+       smpboot_control = smp_processor_id();
 #endif
        initial_code = (unsigned long)wakeup_long64;
-       saved_magic = 0x123456789abcdef0L;
+       saved_magic = 0x123456789abcdef0L;
 #endif /* CONFIG_64BIT */
 
        /*

diff --git a/arch/x86/kernel/asm-offsets.c b/arch/x86/kernel/asm-offsets.c
index 82c783da16a863d9b0318174ef96b076866700c1..797ae1a15c91e3ae7e30080e0b4f292bf1cef8e3 100644 (file)
--- a/arch/x86/kernel/asm-offsets.c
+++ b/arch/x86/kernel/asm-offsets.c
@@ -108,6 +108,7 @@ static void __used common(void)
        OFFSET(TSS_sp1, tss_struct, x86_tss.sp1);
        OFFSET(TSS_sp2, tss_struct, x86_tss.sp2);
        OFFSET(X86_top_of_stack, pcpu_hot, top_of_stack);
+       OFFSET(X86_current_task, pcpu_hot, current_task);
 #ifdef CONFIG_CALL_DEPTH_TRACKING
        OFFSET(X86_call_depth, pcpu_hot, call_depth);
 #endif

diff --git a/arch/x86/kernel/head_64.S b/arch/x86/kernel/head_64.S
index 222efd4a09bc8861b3871f4a175013c0730f8759..5a2417d788d1384e71095dab7fdfe8f1f692e0bb 100644 (file)
--- a/arch/x86/kernel/head_64.S
+++ b/arch/x86/kernel/head_64.S
@@ -61,8 +61,8 @@ SYM_CODE_START_NOALIGN(startup_64)
         * tables and then reload them.
         */
 
-       /* Set up the stack for verify_cpu(), similar to initial_stack below */
-       leaq    (__end_init_task - FRAME_SIZE)(%rip), %rsp
+       /* Set up the stack for verify_cpu() */
+       leaq    (__end_init_task - PTREGS_SIZE)(%rip), %rsp
 
        leaq    _text(%rip), %rdi
 
@@ -241,6 +241,24 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        UNWIND_HINT_EMPTY
        ANNOTATE_NOENDBR // above
 
+#ifdef CONFIG_SMP
+       movl    smpboot_control(%rip), %ecx
+
+       /* Get the per cpu offset for the given CPU# which is in ECX */
+       movq    __per_cpu_offset(,%rcx,8), %rdx
+#else
+       xorl    %edx, %edx
+#endif /* CONFIG_SMP */
+
+       /*
+        * Setup a boot time stack - Any secondary CPU will have lost its stack
+        * by now because the cr3-switch above unmaps the real-mode stack.
+        *
+        * RDX contains the per-cpu offset
+        */
+       movq    pcpu_hot + X86_current_task(%rdx), %rax
+       movq    TASK_threadsp(%rax), %rsp
+
        /*
         * We must switch to a new descriptor in kernel space for the GDT
         * because soon the kernel won't have access anymore to the userspace
@@ -275,12 +293,6 @@ SYM_INNER_LABEL(secondary_startup_64_no_verify, SYM_L_GLOBAL)
        movl    initial_gs+4(%rip),%edx
        wrmsr
 
-       /*
-        * Setup a boot time stack - Any secondary CPU will have lost its stack
-        * by now because the cr3-switch above unmaps the real-mode stack
-        */
-       movq initial_stack(%rip), %rsp
-
        /* Setup and Load IDT */
        pushq   %rsi
        call    early_setup_idt
@@ -372,7 +384,11 @@ SYM_CODE_END(secondary_startup_64)
 SYM_CODE_START(start_cpu0)
        ANNOTATE_NOENDBR
        UNWIND_HINT_EMPTY
-       movq    initial_stack(%rip), %rsp
+
+       /* Find the idle task stack */
+       movq    PER_CPU_VAR(pcpu_hot) + X86_current_task, %rcx
+       movq    TASK_threadsp(%rcx), %rsp
+
        jmp     .Ljump_to_C_code
 SYM_CODE_END(start_cpu0)
 #endif
@@ -420,12 +436,6 @@ SYM_DATA(initial_gs,       .quad INIT_PER_CPU_VAR(fixed_percpu_data))
 #ifdef CONFIG_AMD_MEM_ENCRYPT
 SYM_DATA(initial_vc_handler,   .quad handle_vc_boot_ghcb)
 #endif
-
-/*
- * The FRAME_SIZE gap is a convention which helps the in-kernel unwinder
- * reliably detect the end of the stack.
- */
-SYM_DATA(initial_stack, .quad init_thread_union + THREAD_SIZE - FRAME_SIZE)
        __FINITDATA
 
        __INIT
@@ -660,6 +670,9 @@ SYM_DATA_END(level1_fixmap_pgt)
 SYM_DATA(early_gdt_descr,              .word GDT_ENTRIES*8-1)
 SYM_DATA_LOCAL(early_gdt_descr_base,   .quad INIT_PER_CPU_VAR(gdt_page))
 
+       .align 16
+SYM_DATA(smpboot_control,              .long 0)
+
        .align 16
 /* This must match the first entry in level2_kernel_pgt */
 SYM_DATA(phys_base, .quad 0x0)

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index b18c1385e181b85e302569523bda20a6962aa9d5..62e3bf37f0b8fad6c41bef0bc2599dcbc0ae70b6 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -1112,7 +1112,12 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle,
        idle->thread.sp = (unsigned long)task_pt_regs(idle);
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_rw(cpu);
        initial_code = (unsigned long)start_secondary;
-       initial_stack  = idle->thread.sp;
+
+       if (IS_ENABLED(CONFIG_X86_32)) {
+               initial_stack  = idle->thread.sp;
+       } else {
+               smpboot_control = cpu;
+       }
 
        /* Enable the espfix hack for this CPU */
        init_espfix_ap(cpu);

diff --git a/arch/x86/xen/xen-head.S b/arch/x86/xen/xen-head.S
index ffaa62167f6e8ddf1da9fcc439503e5fa1215700..6bd3914766566e5e3499a3a119c02832327b5b93 100644 (file)
--- a/arch/x86/xen/xen-head.S
+++ b/arch/x86/xen/xen-head.S
@@ -49,7 +49,7 @@ SYM_CODE_START(startup_xen)
        ANNOTATE_NOENDBR
        cld
 
-       mov initial_stack(%rip), %rsp
+       leaq    (__end_init_task - PTREGS_SIZE)(%rip), %rsp
 
        /* Set up %gs.
         *