arm64: entry: simplify trampoline data page

Get rid of some clunky open coded arithmetic on section addresses, by
emitting the trampoline data variables into a separate, dedicated r/o
data section, and putting it at the next page boundary. This way, we can
access the literals via single LDR instruction.

While at it, get rid of other, implicit literals, and use ADRP/ADD or
MOVZ/MOVK sequences, as appropriate. Note that the latter are only
supported for CONFIG_RELOCATABLE=n (which is usually the case if
CONFIG_RANDOMIZE_BASE=n), so update the CPP conditionals to reflect
this.

Acked-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20220622161010.3845775-1-ardb@kernel.org
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h
index daff882883f92c956775b3a8cf15f560f2de2d56..71ed5fdf718bd0fdc49cda4dde69e43d0339547f 100644 (file)
--- a/arch/arm64/include/asm/fixmap.h
+++ b/arch/arm64/include/asm/fixmap.h
@@ -62,10 +62,12 @@ enum fixed_addresses {
 #endif /* CONFIG_ACPI_APEI_GHES */
 
 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
+#ifdef CONFIG_RELOCATABLE
+       FIX_ENTRY_TRAMP_TEXT4,  /* one extra slot for the data page */
+#endif
        FIX_ENTRY_TRAMP_TEXT3,
        FIX_ENTRY_TRAMP_TEXT2,
        FIX_ENTRY_TRAMP_TEXT1,
-       FIX_ENTRY_TRAMP_DATA,
 #define TRAMP_VALIAS           (__fix_to_virt(FIX_ENTRY_TRAMP_TEXT1))
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
        __end_of_permanent_fixed_addresses,

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 5b82b9292400539a627918eb41acae0b866357a9..254fe31c03a07d048c41e2156a95aff997d68d06 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -636,18 +636,28 @@ alternative_else_nop_endif
         */
        .endm
 
-       .macro tramp_data_page  dst
-       adr_l   \dst, .entry.tramp.text
-       sub     \dst, \dst, PAGE_SIZE
-       .endm
-
-       .macro tramp_data_read_var      dst, var
-#ifdef CONFIG_RANDOMIZE_BASE
-       tramp_data_page         \dst
-       add     \dst, \dst, #:lo12:__entry_tramp_data_\var
-       ldr     \dst, [\dst]
+       .macro          tramp_data_read_var     dst, var
+#ifdef CONFIG_RELOCATABLE
+       ldr             \dst, .L__tramp_data_\var
+       .ifndef         .L__tramp_data_\var
+       .pushsection    ".entry.tramp.rodata", "a", %progbits
+       .align          3
+.L__tramp_data_\var:
+       .quad           \var
+       .popsection
+       .endif
 #else
-       ldr     \dst, =\var
+       /*
+        * As !RELOCATABLE implies !RANDOMIZE_BASE the address is always a
+        * compile time constant (and hence not secret and not worth hiding).
+        *
+        * As statically allocated kernel code and data always live in the top
+        * 47 bits of the address space we can sign-extend bit 47 and avoid an
+        * instruction to load the upper 16 bits (which must be 0xFFFF).
+        */
+       movz            \dst, :abs_g2_s:\var
+       movk            \dst, :abs_g1_nc:\var
+       movk            \dst, :abs_g0_nc:\var
 #endif
        .endm
 
@@ -695,7 +705,7 @@ alternative_else_nop_endif
        msr     vbar_el1, x30
        isb
        .else
-       ldr     x30, =vectors
+       adr_l   x30, vectors
        .endif // \kpti == 1
 
        .if     \bhb == BHB_MITIGATION_FW
@@ -764,24 +774,7 @@ SYM_CODE_END(tramp_exit_native)
 SYM_CODE_START(tramp_exit_compat)
        tramp_exit      32
 SYM_CODE_END(tramp_exit_compat)
-
-       .ltorg
        .popsection                             // .entry.tramp.text
-#ifdef CONFIG_RANDOMIZE_BASE
-       .pushsection ".rodata", "a"
-       .align PAGE_SHIFT
-SYM_DATA_START(__entry_tramp_data_start)
-__entry_tramp_data_vectors:
-       .quad   vectors
-#ifdef CONFIG_ARM_SDE_INTERFACE
-__entry_tramp_data___sdei_asm_handler:
-       .quad   __sdei_asm_handler
-#endif /* CONFIG_ARM_SDE_INTERFACE */
-__entry_tramp_data_this_cpu_vector:
-       .quad   this_cpu_vector
-SYM_DATA_END(__entry_tramp_data_start)
-       .popsection                             // .rodata
-#endif /* CONFIG_RANDOMIZE_BASE */
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 
 /*
@@ -932,7 +925,6 @@ NOKPROBE(call_on_irq_stack)
  * This clobbers x4, __sdei_handler() will restore this from firmware's
  * copy.
  */
-.ltorg
 .pushsection ".entry.tramp.text", "ax"
 SYM_CODE_START(__sdei_asm_entry_trampoline)
        mrs     x4, ttbr1_el1
@@ -967,7 +959,6 @@ SYM_CODE_START(__sdei_asm_exit_trampoline)
 1:     sdei_handler_exit exit_mode=x2
 SYM_CODE_END(__sdei_asm_exit_trampoline)
 NOKPROBE(__sdei_asm_exit_trampoline)
-       .ltorg
 .popsection            // .entry.tramp.text
 #endif /* CONFIG_UNMAP_KERNEL_AT_EL0 */
 

diff --git a/arch/arm64/kernel/vmlinux.lds.S b/arch/arm64/kernel/vmlinux.lds.S
index 2d4a8f99517533777371d6ec951ad3c0d7f10eca..8a078c0ee140ce86fad4f4cf39384b6eb7e6d7a5 100644 (file)
--- a/arch/arm64/kernel/vmlinux.lds.S
+++ b/arch/arm64/kernel/vmlinux.lds.S
@@ -115,7 +115,8 @@ jiffies = jiffies_64;
        __entry_tramp_text_start = .;                   \
        *(.entry.tramp.text)                            \
        . = ALIGN(PAGE_SIZE);                           \
-       __entry_tramp_text_end = .;
+       __entry_tramp_text_end = .;                     \
+       *(.entry.tramp.rodata)
 #else
 #define TRAMP_TEXT
 #endif

diff --git a/arch/arm64/mm/mmu.c b/arch/arm64/mm/mmu.c
index 5e2feb0c442da96df26adb85c22f07fddcad44f0..c5563ff990dab6fd7c2daa8f6520e34eb71fd0ea 100644 (file)
--- a/arch/arm64/mm/mmu.c
+++ b/arch/arm64/mm/mmu.c
@@ -672,13 +672,9 @@ static int __init map_entry_trampoline(void)
                __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
                             pa_start + i * PAGE_SIZE, prot);
 
-       if (IS_ENABLED(CONFIG_RANDOMIZE_BASE)) {
-               extern char __entry_tramp_data_start[];
-
-               __set_fixmap(FIX_ENTRY_TRAMP_DATA,
-                            __pa_symbol(__entry_tramp_data_start),
-                            PAGE_KERNEL_RO);
-       }
+       if (IS_ENABLED(CONFIG_RELOCATABLE))
+               __set_fixmap(FIX_ENTRY_TRAMP_TEXT1 - i,
+                            pa_start + i * PAGE_SIZE, PAGE_KERNEL_RO);
 
        return 0;
 }