lazy tlb: allow lazy tlb mm refcounting to be configurable

Add CONFIG_MMU_TLB_REFCOUNT which enables refcounting of the lazy tlb mm
when it is context switched.  This can be disabled by architectures that
don't require this refcounting if they clean up lazy tlb mms when the last
refcount is dropped.  Currently this is always enabled, which is what
existing code does, so the patch is effectively a no-op.

Rename rq->prev_mm to rq->prev_lazy_mm, because that's what it is.

Link: https://lkml.kernel.org/r/20210605014216.446867-3-npiggin@gmail.com
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Cc: Andy Lutomirski <luto@kernel.org>
Cc: Anton Blanchard <anton@ozlabs.org>
Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org>
Cc: Paul Mackerras <paulus@ozlabs.org>
Cc: Randy Dunlap <rdunlap@infradead.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Stephen Rothwell <sfr@canb.auug.org.au>

diff --git a/arch/Kconfig b/arch/Kconfig
index 129df498a8e12c5d6e7da2ac14f9da3811e3d827..9dddd38d2bf41f0329fc024865f40e61600c1be4 100644 (file)
--- a/arch/Kconfig
+++ b/arch/Kconfig
@@ -425,6 +425,10 @@ config ARCH_WANT_IRQS_OFF_ACTIVATE_MM
          irqs disabled over activate_mm. Architectures that do IPI based TLB
          shootdowns should enable this.
 
+# Use normal mm refcounting for MMU_LAZY_TLB kernel thread references.
+config MMU_LAZY_TLB_REFCOUNT
+       def_bool y
+
 config ARCH_HAVE_NMI_SAFE_CMPXCHG
        bool
 

diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index f7a0b347fecbd0c47e6af8b7ae642bfe2f7046b9..939dee9f5bc834cf217c42300854098a97720fad 100644 (file)
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -52,12 +52,21 @@ static inline void mmdrop(struct mm_struct *mm)
 /* Helpers for lazy TLB mm refcounting */
 static inline void mmgrab_lazy_tlb(struct mm_struct *mm)
 {
-       mmgrab(mm);
+       if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT))
+               mmgrab(mm);
 }
 
 static inline void mmdrop_lazy_tlb(struct mm_struct *mm)
 {
-       mmdrop(mm);
+       if (IS_ENABLED(CONFIG_MMU_LAZY_TLB_REFCOUNT)) {
+               mmdrop(mm);
+       } else {
+               /*
+                * mmdrop_lazy_tlb must provide a full memory barrier, see the
+                * membarrier comment finish_task_switch which relies on this.
+                */
+               smp_mb();
+       }
 }
 
 /**

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 030348d3e6d290e8bbd9dc39c119caabaf5136c3..df4002813f023d7b48080b052419a7a1fb6a702b 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -4513,7 +4513,7 @@ static struct rq *finish_task_switch(struct task_struct *prev)
        __releases(rq->lock)
 {
        struct rq *rq = this_rq();
-       struct mm_struct *mm = rq->prev_mm;
+       struct mm_struct *mm = NULL;
        long prev_state;
 
        /*
@@ -4532,7 +4532,10 @@ static struct rq *finish_task_switch(struct task_struct *prev)
                      current->comm, current->pid, preempt_count()))
                preempt_count_set(FORK_PREEMPT_COUNT);
 
-       rq->prev_mm = NULL;
+#ifdef CONFIG_MMU_LAZY_TLB_REFCOUNT
+       mm = rq->prev_lazy_mm;
+       rq->prev_lazy_mm = NULL;
+#endif
 
        /*
         * A task struct has one reference for the use as "current".
@@ -4668,9 +4671,20 @@ context_switch(struct rq *rq, struct task_struct *prev,
                switch_mm_irqs_off(prev->active_mm, next->mm, next);
 
                if (!prev->mm) {                        // from kernel
-                       /* will mmdrop_lazy_tlb() in finish_task_switch(). */
-                       rq->prev_mm = prev->active_mm;
+#ifdef CONFIG_MMU_LAZY_TLB_REFCOUNT
+                       /* Will mmdrop_lazy_tlb() in finish_task_switch(). */
+                       rq->prev_lazy_mm = prev->active_mm;
                        prev->active_mm = NULL;
+#else
+                       /*
+                        * Without MMU_LAZY_TLB_REFCOUNT there is no lazy
+                        * tracking (because no rq->prev_lazy_mm) in
+                        * finish_task_switch, so no mmdrop_lazy_tlb(), so no
+                        * memory barrier for membarrier (see the membarrier
+                        * comment in finish_task_switch()).  Do it here.
+                        */
+                       smp_mb();
+#endif
                }
        }
 

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 14a41a243f7baf308cfbda57c3d3a8c6b3e0d9a3..1a13bb32eba2c54a253a491117af8658fe87e32f 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -967,7 +967,9 @@ struct rq {
        struct task_struct      *idle;
        struct task_struct      *stop;
        unsigned long           next_balance;
-       struct mm_struct        *prev_mm;
+#ifdef CONFIG_MMU_LAZY_TLB_REFCOUNT
+       struct mm_struct        *prev_lazy_mm;
+#endif
 
        unsigned int            clock_update_flags;
        u64                     clock;