arm64: head: record the MMU state at primary entry

Prepare for being able to deal with primary entry with the MMU and
caches enabled, by recording whether or not we entered with the MMU on
in register x19 and in a global variable. (Note that setting this
variable to '1' does not require cache invalidation, nor is it required
for storing the bootargs in that case, so omit the cache maintenance).

Since boot with the MMU and caches enabled is not permitted by the bare
metal boot protocol, ensure that a diagnostic is emitted and a taint bit
set if the MMU was found to be enabled on a non-EFI boot, and panic()
once the console is likely to be up. We will make an exception for EFI
boot later, which has strict requirements for the mapping of system
memory, permitting us to relax the boot protocol and hand over from the
EFI stub to the core kernel with MMU and caches left enabled.

While at it, add 'pre_disable_mmu_workaround' macro invocations to
init_kernel_el, as its manipulation of SCTLR_ELx may amount to disabling
of the MMU after subsequent patches.

Signed-off-by: Ard Biesheuvel <ardb@kernel.org>
Link: https://lore.kernel.org/r/20230111102236.1430401-4-ardb@kernel.org
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index bec97aad092c2b43f6b0ffc2ede5ffb5760124a8..c3b898efd3b5288d098b4d9571f5b6d04f085c90 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -77,6 +77,7 @@
         * primary lowlevel boot path:
         *
         *  Register   Scope                      Purpose
+        *  x19        primary_entry() .. start_kernel()        whether we entered with the MMU on
         *  x20        primary_entry() .. __primary_switch()    CPU boot mode
         *  x21        primary_entry() .. start_kernel()        FDT pointer passed at boot in x0
         *  x22        create_idmap() .. start_kernel()         ID map VA of the DT blob
@@ -86,6 +87,7 @@
         *  x28        create_idmap()                           callee preserved temp register
         */
 SYM_CODE_START(primary_entry)
+       bl      record_mmu_state
        bl      preserve_boot_args
        bl      init_kernel_el                  // w0=cpu_boot_mode
        mov     x20, x0
@@ -109,6 +111,18 @@ SYM_CODE_START(primary_entry)
        b       __primary_switch
 SYM_CODE_END(primary_entry)
 
+SYM_CODE_START_LOCAL(record_mmu_state)
+       mrs     x19, CurrentEL
+       cmp     x19, #CurrentEL_EL2
+       mrs     x19, sctlr_el1
+       b.ne    0f
+       mrs     x19, sctlr_el2
+0:     tst     x19, #SCTLR_ELx_C               // Z := (C == 0)
+       and     x19, x19, #SCTLR_ELx_M          // isolate M bit
+       csel    x19, xzr, x19, eq               // clear x19 if Z
+       ret
+SYM_CODE_END(record_mmu_state)
+
 /*
  * Preserve the arguments passed by the bootloader in x0 .. x3
  */
@@ -119,11 +133,14 @@ SYM_CODE_START_LOCAL(preserve_boot_args)
        stp     x21, x1, [x0]                   // x0 .. x3 at kernel entry
        stp     x2, x3, [x0, #16]
 
+       cbnz    x19, 0f                         // skip cache invalidation if MMU is on
        dmb     sy                              // needed before dc ivac with
                                                // MMU off
 
        add     x1, x0, #0x20                   // 4 x 8 bytes
        b       dcache_inval_poc                // tail call
+0:     str_l   x19, mmu_enabled_at_boot, x0
+       ret
 SYM_CODE_END(preserve_boot_args)
 
 SYM_FUNC_START_LOCAL(clear_page_tables)
@@ -497,6 +514,7 @@ SYM_FUNC_START(init_kernel_el)
 
 SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
        mov_q   x0, INIT_SCTLR_EL1_MMU_OFF
+       pre_disable_mmu_workaround
        msr     sctlr_el1, x0
        isb
        mov_q   x0, INIT_PSTATE_EL1
@@ -529,11 +547,13 @@ SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
        cbz     x0, 1f
 
        /* Set a sane SCTLR_EL1, the VHE way */
+       pre_disable_mmu_workaround
        msr_s   SYS_SCTLR_EL12, x1
        mov     x2, #BOOT_CPU_FLAG_E2H
        b       2f
 
 1:
+       pre_disable_mmu_workaround
        msr     sctlr_el1, x1
        mov     x2, xzr
 2:

diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c
index 12cfe9d0d3fac10d7885335d43119f9788f96da6..b8ec7b3ac9cbe8a8c207aab0b385926d910c45b1 100644 (file)
--- a/arch/arm64/kernel/setup.c
+++ b/arch/arm64/kernel/setup.c
@@ -58,6 +58,7 @@ static int num_standard_resources;
 static struct resource *standard_resources;
 
 phys_addr_t __fdt_pointer __initdata;
+u64 mmu_enabled_at_boot __initdata;
 
 /*
  * Standard memory resources
@@ -332,8 +333,12 @@ void __init __no_sanitize_address setup_arch(char **cmdline_p)
        xen_early_init();
        efi_init();
 
-       if (!efi_enabled(EFI_BOOT) && ((u64)_text % MIN_KIMG_ALIGN) != 0)
-            pr_warn(FW_BUG "Kernel image misaligned at boot, please fix your bootloader!");
+       if (!efi_enabled(EFI_BOOT)) {
+               if ((u64)_text % MIN_KIMG_ALIGN)
+                       pr_warn(FW_BUG "Kernel image misaligned at boot, please fix your bootloader!");
+               WARN_TAINT(mmu_enabled_at_boot, TAINT_FIRMWARE_WORKAROUND,
+                          FW_BUG "Booted with MMU enabled!");
+       }
 
        arm64_memblock_init();
 
@@ -442,3 +447,11 @@ static int __init register_arm64_panic_block(void)
        return 0;
 }
 device_initcall(register_arm64_panic_block);
+
+static int __init check_mmu_enabled_at_boot(void)
+{
+       if (!efi_enabled(EFI_BOOT) && mmu_enabled_at_boot)
+               panic("Non-EFI boot detected with MMU and caches enabled");
+       return 0;
+}
+device_initcall_sync(check_mmu_enabled_at_boot);