x86/kexec: Cope with relocate_kernel() not being at the start of the page

A few places in the kexec control code page make the assumption that the first
instruction of relocate_kernel is at the very start of the page.

To allow for Clang CFI information to be added to relocate_kernel(), as well
as the general principle of removing unwarranted assumptions, fix them to use
the external __relocate_kernel_start symbol that the linker adds. This means
using a separate addq and subq for calculating offsets, as the assembler can
no longer calculate the delta directly for itself and relocations aren't that
versatile. But those values can at least be used relative to a local label to
avoid absolute relocations.

Turn the jump from relocate_kernel() to identity_mapped() into a real indirect
'jmp *%rsi' too, while touching it. There was no real reason for it to be
a push+ret in the first place, and adding Clang CFI info will also give
objtool enough visibility to start complaining 'return with modified stack
frame' about it.

  [ bp: Massage commit message. ]

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Signed-off-by: Borislav Petkov (AMD) <bp@alien8.de>
Link: https://lore.kernel.org/r/20250109140757.2841269-9-dwmw2@infradead.org

diff --git a/arch/x86/kernel/relocate_kernel_64.S b/arch/x86/kernel/relocate_kernel_64.S
index 14ed40b7f322e1d5e3f78368e773327cf8a36180..af2cd06ff318d65ba65cf96ab4bdecf98bdb2cec 100644 (file)
--- a/arch/x86/kernel/relocate_kernel_64.S
+++ b/arch/x86/kernel/relocate_kernel_64.S
@@ -95,11 +95,10 @@ SYM_CODE_START_NOALIGN(relocate_kernel)
        lea     PAGE_SIZE(%rsi), %rsp
 
        /* jump to identity mapped page */
-       addq    $(identity_mapped - relocate_kernel), %rsi
-       pushq   %rsi
-       ANNOTATE_UNRET_SAFE
-       ret
-       int3
+0:     addq    $identity_mapped - 0b, %rsi
+       subq    $__relocate_kernel_start - 0b, %rsi
+       ANNOTATE_RETPOLINE_SAFE
+       jmp     *%rsi
 SYM_CODE_END(relocate_kernel)
 
 SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
@@ -219,16 +218,21 @@ SYM_CODE_START_LOCAL_NOALIGN(identity_mapped)
 
        /* get the re-entry point of the peer system */
        popq    %rbp
-       leaq    relocate_kernel(%rip), %r8
        movq    kexec_pa_swap_page(%rip), %r10
        movq    pa_backup_pages_map(%rip), %rdi
        movq    kexec_pa_table_page(%rip), %rax
        movq    %rax, %cr3
+
+       /* Find start (and end) of this physical mapping of control page */
+       leaq    (%rip), %r8
+       ANNOTATE_NOENDBR
+       andq    $PAGE_MASK, %r8
        lea     PAGE_SIZE(%r8), %rsp
        movl    $1, %r11d       /* Ensure preserve_context flag is set */
        call    swap_pages
        movq    kexec_va_control_page(%rip), %rax
-       addq    $(virtual_mapped - relocate_kernel), %rax
+0:     addq    $virtual_mapped - 0b, %rax
+       subq    $__relocate_kernel_start - 0b, %rax
        pushq   %rax
        ANNOTATE_UNRET_SAFE
        ret