arm64: Fix interrupt tracing in the presence of NMIs

In the presence of any form of instrumentation, nmi_enter() should be
done before calling any traceable code and any instrumentation code.

Currently, nmi_enter() is done in handle_domain_nmi(), which is much
too late as instrumentation code might get called before. Move the
nmi_enter/exit() calls to the arch IRQ vector handler.

On arm64, it is not possible to know if the IRQ vector handler was
called because of an NMI before acknowledging the interrupt. However, It
is possible to know whether normal interrupts could be taken in the
interrupted context (i.e. if taking an NMI in that context could
introduce a potential race condition).

When interrupting a context with IRQs disabled, call nmi_enter() as soon
as possible. In contexts with IRQs enabled, defer this to the interrupt
controller, which is in a better position to know if an interrupt taken
is an NMI.

Fixes: bc3c03ccb464 ("arm64: Enable the support of pseudo-NMIs")
Cc: <stable@vger.kernel.org> # 5.1.x-
Cc: Will Deacon <will.deacon@arm.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Julien Thierry <julien.thierry@arm.com>
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>

diff --git a/arch/arm64/kernel/entry.S b/arch/arm64/kernel/entry.S
index 89ab6bd896c41e69cd2112ee31e2a56c0ef01152..6d5966346710164d3ddb69b48857b7e547103fca 100644 (file)
--- a/arch/arm64/kernel/entry.S
+++ b/arch/arm64/kernel/entry.S
@@ -435,6 +435,20 @@ tsk        .req    x28             // current thread_info
        irq_stack_exit
        .endm
 
+#ifdef CONFIG_ARM64_PSEUDO_NMI
+       /*
+        * Set res to 0 if irqs were unmasked in interrupted context.
+        * Otherwise set res to non-0 value.
+        */
+       .macro  test_irqs_unmasked res:req, pmr:req
+alternative_if ARM64_HAS_IRQ_PRIO_MASKING
+       sub     \res, \pmr, #GIC_PRIO_IRQON
+alternative_else
+       mov     \res, xzr
+alternative_endif
+       .endm
+#endif
+
        .text
 
 /*
@@ -631,19 +645,19 @@ ENDPROC(el1_sync)
 el1_irq:
        kernel_entry 1
        enable_da_f
-#ifdef CONFIG_TRACE_IRQFLAGS
+
 #ifdef CONFIG_ARM64_PSEUDO_NMI
 alternative_if ARM64_HAS_IRQ_PRIO_MASKING
        ldr     x20, [sp, #S_PMR_SAVE]
-alternative_else
-       mov     x20, #GIC_PRIO_IRQON
-alternative_endif
-       cmp     x20, #GIC_PRIO_IRQOFF
-       /* Irqs were disabled, don't trace */
-       b.ls    1f
+alternative_else_nop_endif
+       test_irqs_unmasked      res=x0, pmr=x20
+       cbz     x0, 1f
+       bl      asm_nmi_enter
+1:
 #endif
+
+#ifdef CONFIG_TRACE_IRQFLAGS
        bl      trace_hardirqs_off
-1:
 #endif
 
        irq_handler
@@ -662,14 +676,22 @@ alternative_else_nop_endif
        bl      preempt_schedule_irq            // irq en/disable is done inside
 1:
 #endif
-#ifdef CONFIG_TRACE_IRQFLAGS
+
 #ifdef CONFIG_ARM64_PSEUDO_NMI
        /*
         * if IRQs were disabled when we received the interrupt, we have an NMI
         * and we are not re-enabling interrupt upon eret. Skip tracing.
         */
-       cmp     x20, #GIC_PRIO_IRQOFF
-       b.ls    1f
+       test_irqs_unmasked      res=x0, pmr=x20
+       cbz     x0, 1f
+       bl      asm_nmi_exit
+1:
+#endif
+
+#ifdef CONFIG_TRACE_IRQFLAGS
+#ifdef CONFIG_ARM64_PSEUDO_NMI
+       test_irqs_unmasked      res=x0, pmr=x20
+       cbnz    x0, 1f
 #endif
        bl      trace_hardirqs_on
 1:

diff --git a/arch/arm64/kernel/irq.c b/arch/arm64/kernel/irq.c
index 92fa81798fb9ab0458ba81e850fb18fc4c8fd854..fdd9cb27fed5517f25e020f785affee4c6ecc75e 100644 (file)
--- a/arch/arm64/kernel/irq.c
+++ b/arch/arm64/kernel/irq.c
@@ -27,8 +27,10 @@
 #include <linux/smp.h>
 #include <linux/init.h>
 #include <linux/irqchip.h>
+#include <linux/kprobes.h>
 #include <linux/seq_file.h>
 #include <linux/vmalloc.h>
+#include <asm/daifflags.h>
 #include <asm/vmap_stack.h>
 
 unsigned long irq_err_count;
@@ -76,3 +78,18 @@ void __init init_IRQ(void)
        if (!handle_arch_irq)
                panic("No interrupt controller found.");
 }
+
+/*
+ * Stubs to make nmi_enter/exit() code callable from ASM
+ */
+asmlinkage void notrace asm_nmi_enter(void)
+{
+       nmi_enter();
+}
+NOKPROBE_SYMBOL(asm_nmi_enter);
+
+asmlinkage void notrace asm_nmi_exit(void)
+{
+       nmi_exit();
+}
+NOKPROBE_SYMBOL(asm_nmi_exit);

diff --git a/drivers/irqchip/irq-gic-v3.c b/drivers/irqchip/irq-gic-v3.c
index f44cd89cfc40e031cdabd305d60b222a53fad81d..b176700bb3878cfb827a8bda643e1bb2c84474ac 100644 (file)
--- a/drivers/irqchip/irq-gic-v3.c
+++ b/drivers/irqchip/irq-gic-v3.c
@@ -472,8 +472,12 @@ static void gic_deactivate_unhandled(u32 irqnr)
 
 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
 {
+       bool irqs_enabled = interrupts_enabled(regs);
        int err;
 
+       if (irqs_enabled)
+               nmi_enter();
+
        if (static_branch_likely(&supports_deactivate_key))
                gic_write_eoir(irqnr);
        /*
@@ -485,6 +489,9 @@ static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs)
        err = handle_domain_nmi(gic_data.domain, irqnr, regs);
        if (err)
                gic_deactivate_unhandled(irqnr);
+
+       if (irqs_enabled)
+               nmi_exit();
 }
 
 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)

diff --git a/kernel/irq/irqdesc.c b/kernel/irq/irqdesc.c
index c52b737ab8e3183c4051c8516406ba5cec194aa3..a92b33593b8d0c80f534cabef96b0fc1acd65427 100644 (file)
--- a/kernel/irq/irqdesc.c
+++ b/kernel/irq/irqdesc.c
@@ -680,6 +680,8 @@ int __handle_domain_irq(struct irq_domain *domain, unsigned int hwirq,
  * @hwirq:     The HW irq number to convert to a logical one
  * @regs:      Register file coming from the low-level handling code
  *
+ *             This function must be called from an NMI context.
+ *
  * Returns:    0 on success, or -EINVAL if conversion has failed
  */
 int handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq,
@@ -689,7 +691,10 @@ int handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq,
        unsigned int irq;
        int ret = 0;
 
-       nmi_enter();
+       /*
+        * NMI context needs to be setup earlier in order to deal with tracing.
+        */
+       WARN_ON(!in_nmi());
 
        irq = irq_find_mapping(domain, hwirq);
 
@@ -702,7 +707,6 @@ int handle_domain_nmi(struct irq_domain *domain, unsigned int hwirq,
        else
                ret = -EINVAL;
 
-       nmi_exit();
        set_irq_regs(old_regs);
        return ret;
 }