arm64: entry: Simplify tramp_alias macro and tramp_exit routine

The tramp_alias macro constructs the virtual alias of a symbol in the
trampoline text mapping, based on its kernel text address, and does so
in a way that is more convoluted than necessary. So let's simplify that.

Also, now that the address of the vector table is kept in a per-CPU
variable, there is no need to defer the load and the assignment of
VBAR_EL1 to tramp_exit(). This means we can use a PC-relative reference
to the per-CPU variable instead of storing its absolute address in a
global variable in the trampoline rodata.

And given that tramp_alias no longer needs a temp register, this means
we can restore X30 earlier as well, and only leave X29 for tramp_exit()
to restore.

While at it, give some related symbols static linkage, considering that
they are only referenced from the object file that defines them.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Reviewed-by: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20230418143604.1176437-3-ardb@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 16fbd0d9790dd436397d7f7a3bc56dbb93f4f233..a40e5e50fa55232e1e06557daca255ae3741cd5f 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -101,12 +101,11 @@
 .org .Lventry_start\@ + 128    // Did we overflow the ventry slot?
        .endm
 
-       .macro tramp_alias, dst, sym, tmp
-       mov_q   \dst, TRAMP_VALIAS
-       adr_l   \tmp, \sym
-       add     \dst, \dst, \tmp
-       adr_l   \tmp, .entry.tramp.text
-       sub     \dst, \dst, \tmp
+       .macro  tramp_alias, dst, sym
+       .set    .Lalias\@, TRAMP_VALIAS + \sym - .entry.tramp.text
+       movz    \dst, :abs_g2_s:.Lalias\@
+       movk    \dst, :abs_g1_nc:.Lalias\@
+       movk    \dst, :abs_g0_nc:.Lalias\@
        .endm
 
        /*
@@ -436,8 +435,13 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
 alternative_else_nop_endif
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
        msr     far_el1, x29
-       tramp_alias     x30, tramp_exit_native, x29
-       br      x30
+
+       ldr_this_cpu    x30, this_cpu_vector, x29
+       tramp_alias     x29, tramp_exit
+       msr             vbar_el1, x30           // install vector table
+       ldr             lr, [sp, #S_LR]         // restore x30
+       add             sp, sp, #PT_REGS_SIZE   // restore sp
+       br              x29
 #endif
        .else
        ldr     lr, [sp, #S_LR]
@@ -728,20 +732,6 @@ alternative_else_nop_endif
 .org 1b + 128  // Did we overflow the ventry slot?
        .endm
 
-       .macro tramp_exit, regsize = 64
-       tramp_data_read_var     x30, this_cpu_vector
-       get_this_cpu_offset x29
-       ldr     x30, [x30, x29]
-
-       msr     vbar_el1, x30
-       ldr     lr, [sp, #S_LR]
-       tramp_unmap_kernel      x29
-       mrs     x29, far_el1
-       add     sp, sp, #PT_REGS_SIZE           // restore sp
-       eret
-       sb
-       .endm
-
        .macro  generate_tramp_vector,  kpti, bhb
 .Lvector_start\@:
        .space  0x400
@@ -762,7 +752,7 @@ alternative_else_nop_endif
  */
        .pushsection ".entry.tramp.text", "ax"
        .align  11
-SYM_CODE_START_NOALIGN(tramp_vectors)
+SYM_CODE_START_LOCAL_NOALIGN(tramp_vectors)
 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_LOOP
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_FW
@@ -771,9 +761,12 @@ SYM_CODE_START_NOALIGN(tramp_vectors)
        generate_tramp_vector   kpti=1, bhb=BHB_MITIGATION_NONE
 SYM_CODE_END(tramp_vectors)
 
-SYM_CODE_START(tramp_exit_native)
-       tramp_exit
-SYM_CODE_END(tramp_exit_native)
+SYM_CODE_START_LOCAL(tramp_exit)
+       tramp_unmap_kernel      x29
+       mrs             x29, far_el1            // restore x29
+       eret
+       sb
+SYM_CODE_END(tramp_exit)
        .popsection                             // .entry.tramp.text
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
@@ -1067,7 +1060,7 @@ alternative_if_not ARM64_UNMAP_KERNEL_AT_EL0
 alternative_else_nop_endif
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
-       tramp_alias     dst=x5, sym=__sdei_asm_exit_trampoline, tmp=x3
+       tramp_alias     dst=x5, sym=__sdei_asm_exit_trampoline
        br      x5
 #endif
 SYM_CODE_END(__sdei_asm_handler)