arm64: Initialise as nVHE before switching to VHE

As we are aiming to be able to control whether we enable VHE or
not, let's always drop down to EL1 first, and only then upgrade
to VHE if at all possible.

This means that if the kernel is booted at EL2, we always start
with a nVHE init, drop to EL1 to initialise the the kernel, and
only then upgrade the kernel EL to EL2 if possible (the process
is obviously shortened for secondary CPUs).

The resume path is handled similarly to a secondary CPU boot.

Signed-off-by: Marc Zyngier <maz@kernel.org>
Acked-by: David Brazdil <dbrazdil@google.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Link: https://lore.kernel.org/r/20210208095732.3267263-6-maz@kernel.org
[will: Avoid calling switch_to_vhe twice on kaslr path]
Signed-off-by: Will Deacon <will@kernel.org>

diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S
index 28e9735302dffd6f24bf1efdf5a2946b97c81d75..ec66dc061b0c75afa4a0f1806be14d7ecada355e 100644 (file)
--- a/arch/arm64/kernel/head.S
+++ b/arch/arm64/kernel/head.S
@@ -447,6 +447,7 @@ SYM_FUNC_START_LOCAL(__primary_switched)
        ret                                     // to __primary_switch()
 0:
 #endif
+       bl      switch_to_vhe                   // Prefer VHE if possible
        add     sp, sp, #16
        mov     x29, #0
        mov     x30, #0
@@ -493,42 +494,6 @@ SYM_INNER_LABEL(init_el1, SYM_L_LOCAL)
        eret
 
 SYM_INNER_LABEL(init_el2, SYM_L_LOCAL)
-#ifdef CONFIG_ARM64_VHE
-       /*
-        * Check for VHE being present. x2 being non-zero indicates that we
-        * do have VHE, and that the kernel is intended to run at EL2.
-        */
-       mrs     x2, id_aa64mmfr1_el1
-       ubfx    x2, x2, #ID_AA64MMFR1_VHE_SHIFT, #4
-#else
-       mov     x2, xzr
-#endif
-       cbz     x2, init_el2_nvhe
-
-       /*
-        * When VHE _is_ in use, EL1 will not be used in the host and
-        * requires no configuration, and all non-hyp-specific EL2 setup
-        * will be done via the _EL1 system register aliases in __cpu_setup.
-        */
-       mov_q   x0, HCR_HOST_VHE_FLAGS
-       msr     hcr_el2, x0
-       isb
-
-       init_el2_state vhe
-
-       isb
-
-       mov_q   x0, INIT_PSTATE_EL2
-       msr     spsr_el2, x0
-       msr     elr_el2, lr
-       mov     w0, #BOOT_CPU_MODE_EL2
-       eret
-
-SYM_INNER_LABEL(init_el2_nvhe, SYM_L_LOCAL)
-       /*
-        * When VHE is not in use, early init of EL2 and EL1 needs to be
-        * done here.
-        */
        mov_q   x0, INIT_SCTLR_EL1_MMU_OFF
        msr     sctlr_el1, x0
 
@@ -623,6 +588,7 @@ SYM_FUNC_START_LOCAL(secondary_startup)
        /*
         * Common entry point for secondary CPUs.
         */
+       bl      switch_to_vhe
        bl      __cpu_secondary_check52bitva
        bl      __cpu_setup                     // initialise processor
        adrp    x1, swapper_pg_dir

diff --git a/arch/arm64/kernel/hyp-stub.S b/arch/arm64/kernel/hyp-stub.S
index 3f3dbbe8914dd63a4ecc63d35a725110ddf55638..373ed2213e1d9aec3c91cff9516e63dfd295f19d 100644 (file)
--- a/arch/arm64/kernel/hyp-stub.S
+++ b/arch/arm64/kernel/hyp-stub.S
@@ -190,3 +190,27 @@ SYM_FUNC_START(__hyp_reset_vectors)
        hvc     #0
        ret
 SYM_FUNC_END(__hyp_reset_vectors)
+
+/*
+ * Entry point to switch to VHE if deemed capable
+ */
+SYM_FUNC_START(switch_to_vhe)
+#ifdef CONFIG_ARM64_VHE
+       // Need to have booted at EL2
+       adr_l   x1, __boot_cpu_mode
+       ldr     w0, [x1]
+       cmp     w0, #BOOT_CPU_MODE_EL2
+       b.ne    1f
+
+       // and still be at EL1
+       mrs     x0, CurrentEL
+       cmp     x0, #CurrentEL_EL1
+       b.ne    1f
+
+       // Turn the world upside down
+       mov     x0, #HVC_VHE_RESTART
+       hvc     #0
+1:
+#endif
+       ret
+SYM_FUNC_END(switch_to_vhe)

diff --git a/arch/arm64/kernel/sleep.S b/arch/arm64/kernel/sleep.S
index 6bdef7362c0eba40a4cee4c7fb3b9955b41a3287..5bfd9b87f85df784894e9ed2ed576da63f2c3ed1 100644 (file)
--- a/arch/arm64/kernel/sleep.S
+++ b/arch/arm64/kernel/sleep.S
@@ -100,6 +100,7 @@ SYM_FUNC_END(__cpu_suspend_enter)
        .pushsection ".idmap.text", "awx"
 SYM_CODE_START(cpu_resume)
        bl      init_kernel_el
+       bl      switch_to_vhe
        bl      __cpu_setup
        /* enable the MMU early - so we can access sleep_save_stash by va */
        adrp    x1, swapper_pg_dir