softirq: Add RT specific softirq accounting

RT requires the softirq processing and local bottomhalf disabled regions to
be preemptible. Using the normal preempt count based serialization is
therefore not possible because this implicitely disables preemption.

RT kernels use a per CPU local lock to serialize bottomhalfs. As
local_bh_disable() can nest the lock can only be acquired on the outermost
invocation of local_bh_disable() and released when the nest count becomes
zero. Tasks which hold the local lock can be preempted so its required to
keep track of the nest count per task.

Add a RT only counter to task struct and adjust the relevant macros in
preempt.h.

Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Tested-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Tested-by: Paul E. McKenney <paulmck@kernel.org>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Link: https://lore.kernel.org/r/20210309085726.983627589@linutronix.de

diff --git a/include/linux/hardirq.h b/include/linux/hardirq.h
index 7c9d6a2d7e90905ffa267df45a2e4ec040c29a76..69bc86ea382c2d9e503831f9e6b5bffe798519e0 100644 (file)
--- a/include/linux/hardirq.h
+++ b/include/linux/hardirq.h
@@ -6,6 +6,7 @@
 #include <linux/preempt.h>
 #include <linux/lockdep.h>
 #include <linux/ftrace_irq.h>
+#include <linux/sched.h>
 #include <linux/vtime.h>
 #include <asm/hardirq.h>
 

diff --git a/include/linux/preempt.h b/include/linux/preempt.h
index 69cc8b64aa3a0e3b957fff471b15adcd1dbaa5fe..9881eac0698fd8eb25044b47071d54c11813aa67 100644 (file)
--- a/include/linux/preempt.h
+++ b/include/linux/preempt.h
@@ -79,7 +79,11 @@
 
 #define nmi_count()    (preempt_count() & NMI_MASK)
 #define hardirq_count()        (preempt_count() & HARDIRQ_MASK)
-#define softirq_count()        (preempt_count() & SOFTIRQ_MASK)
+#ifdef CONFIG_PREEMPT_RT
+# define softirq_count()       (current->softirq_disable_cnt & SOFTIRQ_MASK)
+#else
+# define softirq_count()       (preempt_count() & SOFTIRQ_MASK)
+#endif
 #define irq_count()    (nmi_count() | hardirq_count() | softirq_count())
 
 /*

diff --git a/include/linux/sched.h b/include/linux/sched.h
index ef00bb22164cd60291aa4b3c9761a26e865ef505..743a613c9cf3e85f3d61cabf0e49a523931d803a 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -1044,6 +1044,9 @@ struct task_struct {
        int                             softirq_context;
        int                             irq_config;
 #endif
+#ifdef CONFIG_PREEMPT_RT
+       int                             softirq_disable_cnt;
+#endif
 
 #ifdef CONFIG_LOCKDEP
 # define MAX_LOCK_DEPTH                        48UL