arm64: Allow IPIs to be handled as normal interrupts

In order to deal with IPIs as normal interrupts, let's add
a new way to register them with the architecture code.

set_smp_ipi_range() takes a range of interrupts, and allows
the arch code to request them as if the were normal interrupts.
A standard handler is then called by the core IRQ code to deal
with the IPI.

This means that we don't need to call irq_enter/irq_exit, and
that we don't need to deal with set_irq_regs either. So let's
move the dispatcher into its own function, and leave handle_IPI()
as a compatibility function.

On the sending side, let's make use of ipi_send_mask, which
already exists for this purpose.

One of the major difference is that we end up, in some cases
(such as when performing IRQ time accounting on the scheduler
IPI), end up with nested irq_enter()/irq_exit() pairs.
Other than the (relatively small) overhead, there should be
no consequences to it (these pairs are designed to nest
correctly, and the accounting shouldn't be off).

Reviewed-by: Valentin Schneider <valentin.schneider@arm.com>
Acked-by: Catalin Marinas <catalin.marinas@arm.com>
Signed-off-by: Marc Zyngier <maz@kernel.org>

diff --git a/arch/arm64/Kconfig b/arch/arm64/Kconfig
index 6d232837cbeee8abb0c5a9c37b05c8566260156f..d0fdbe5fb32f0970e41a551e393ceb3ac4747a9d 100644 (file)
--- a/arch/arm64/Kconfig
+++ b/arch/arm64/Kconfig
@@ -106,6 +106,7 @@ config ARM64
        select GENERIC_CPU_VULNERABILITIES
        select GENERIC_EARLY_IOREMAP
        select GENERIC_IDLE_POLL_SETUP
+       select GENERIC_IRQ_IPI
        select GENERIC_IRQ_MULTI_HANDLER
        select GENERIC_IRQ_PROBE
        select GENERIC_IRQ_SHOW

diff --git a/arch/arm64/include/asm/smp.h b/arch/arm64/include/asm/smp.h
index 0eadbf933e3592353d6337db9d85aca681d2c4c5..57c5db15f6b774c7d7550a0299cad814badfcfa0 100644 (file)
--- a/arch/arm64/include/asm/smp.h
+++ b/arch/arm64/include/asm/smp.h
@@ -78,6 +78,11 @@ extern void set_smp_cross_call(void (*)(const struct cpumask *, unsigned int));
 
 extern void (*__smp_cross_call)(const struct cpumask *, unsigned int);
 
+/*
+ * Register IPI interrupts with the arch SMP code
+ */
+extern void set_smp_ipi_range(int ipi_base, int nr_ipi);
+
 /*
  * Called from the secondary holding pen, this is the secondary CPU entry point.
  */

diff --git a/arch/arm64/kernel/smp.c b/arch/arm64/kernel/smp.c
index 355ee9eed4dded61a666edd0944aa14a512128e9..00c9db1b61b59d902ca4217544faef67b7357710 100644 (file)
--- a/arch/arm64/kernel/smp.c
+++ b/arch/arm64/kernel/smp.c
@@ -75,6 +75,13 @@ enum ipi_msg_type {
        IPI_WAKEUP
 };
 
+static int ipi_irq_base __read_mostly;
+static int nr_ipi __read_mostly = NR_IPI;
+static struct irq_desc *ipi_desc[NR_IPI] __read_mostly;
+
+static void ipi_setup(int cpu);
+static void ipi_teardown(int cpu);
+
 #ifdef CONFIG_HOTPLUG_CPU
 static int op_cpu_kill(unsigned int cpu);
 #else
@@ -237,6 +244,8 @@ asmlinkage notrace void secondary_start_kernel(void)
         */
        notify_cpu_starting(cpu);
 
+       ipi_setup(cpu);
+
        store_cpu_topology(cpu);
        numa_add_cpu(cpu);
 
@@ -302,6 +311,7 @@ int __cpu_disable(void)
         * and we must not schedule until we're ready to give up the cpu.
         */
        set_cpu_online(cpu, false);
+       ipi_teardown(cpu);
 
        /*
         * OK - migrate IRQs away from this CPU
@@ -890,10 +900,9 @@ static void ipi_cpu_crash_stop(unsigned int cpu, struct pt_regs *regs)
 /*
  * Main handler for inter-processor interrupts
  */
-void handle_IPI(int ipinr, struct pt_regs *regs)
+static void do_handle_IPI(int ipinr)
 {
        unsigned int cpu = smp_processor_id();
-       struct pt_regs *old_regs = set_irq_regs(regs);
 
        if ((unsigned)ipinr < NR_IPI) {
                trace_ipi_entry_rcuidle(ipi_types[ipinr]);
@@ -906,21 +915,16 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
                break;
 
        case IPI_CALL_FUNC:
-               irq_enter();
                generic_smp_call_function_interrupt();
-               irq_exit();
                break;
 
        case IPI_CPU_STOP:
-               irq_enter();
                local_cpu_stop();
-               irq_exit();
                break;
 
        case IPI_CPU_CRASH_STOP:
                if (IS_ENABLED(CONFIG_KEXEC_CORE)) {
-                       irq_enter();
-                       ipi_cpu_crash_stop(cpu, regs);
+                       ipi_cpu_crash_stop(cpu, get_irq_regs());
 
                        unreachable();
                }
@@ -928,17 +932,13 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 
 #ifdef CONFIG_GENERIC_CLOCKEVENTS_BROADCAST
        case IPI_TIMER:
-               irq_enter();
                tick_receive_broadcast();
-               irq_exit();
                break;
 #endif
 
 #ifdef CONFIG_IRQ_WORK
        case IPI_IRQ_WORK:
-               irq_enter();
                irq_work_run();
-               irq_exit();
                break;
 #endif
 
@@ -957,9 +957,78 @@ void handle_IPI(int ipinr, struct pt_regs *regs)
 
        if ((unsigned)ipinr < NR_IPI)
                trace_ipi_exit_rcuidle(ipi_types[ipinr]);
+}
+
+/* Legacy version, should go away once all irqchips have been converted */
+void handle_IPI(int ipinr, struct pt_regs *regs)
+{
+       struct pt_regs *old_regs = set_irq_regs(regs);
+
+       irq_enter();
+       do_handle_IPI(ipinr);
+       irq_exit();
+
        set_irq_regs(old_regs);
 }
 
+static irqreturn_t ipi_handler(int irq, void *data)
+{
+       do_handle_IPI(irq - ipi_irq_base);
+       return IRQ_HANDLED;
+}
+
+static void ipi_send(const struct cpumask *target, unsigned int ipi)
+{
+       __ipi_send_mask(ipi_desc[ipi], target);
+}
+
+static void ipi_setup(int cpu)
+{
+       int i;
+
+       if (!ipi_irq_base)
+               return;
+
+       for (i = 0; i < nr_ipi; i++)
+               enable_percpu_irq(ipi_irq_base + i, 0);
+}
+
+static void ipi_teardown(int cpu)
+{
+       int i;
+
+       if (!ipi_irq_base)
+               return;
+
+       for (i = 0; i < nr_ipi; i++)
+               disable_percpu_irq(ipi_irq_base + i);
+}
+
+void __init set_smp_ipi_range(int ipi_base, int n)
+{
+       int i;
+
+       WARN_ON(n < NR_IPI);
+       nr_ipi = min(n, NR_IPI);
+
+       for (i = 0; i < nr_ipi; i++) {
+               int err;
+
+               err = request_percpu_irq(ipi_base + i, ipi_handler,
+                                        "IPI", &irq_stat);
+               WARN_ON(err);
+
+               ipi_desc[i] = irq_to_desc(ipi_base + i);
+               irq_set_status_flags(ipi_base + i, IRQ_HIDDEN);
+       }
+
+       ipi_irq_base = ipi_base;
+       __smp_cross_call = ipi_send;
+
+       /* Setup the boot CPU immediately */
+       ipi_setup(smp_processor_id());
+}
+
 void smp_send_reschedule(int cpu)
 {
        smp_cross_call(cpumask_of(cpu), IPI_RESCHEDULE);