mm/mmap: free vm_area_struct without call_rcu in exit_mmap

call_rcu() can take a long time when callback offloading is enabled.  Its
use in the vm_area_free can cause regressions in the exit path when
multiple VMAs are being freed.

Because exit_mmap() is called only after the last mm user drops its
refcount, the page fault handlers can't be racing with it.  Any other
possible user like oom-reaper or process_mrelease are already synchronized
using mmap_lock.  Therefore exit_mmap() can free VMAs directly, without
the use of call_rcu().

Expose __vm_area_free() and use it from exit_mmap() to avoid possible
call_rcu() floods and performance regressions caused by it.

Link: https://lkml.kernel.org/r/20230227173632.3292573-33-surenb@google.com
Signed-off-by: Suren Baghdasaryan <surenb@google.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 82ea2ae374de6078e06359ca749539831e640cce..e666e890eb1857357c945f83b04dfbc3bab60567 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -256,6 +256,8 @@ void setup_initial_init_mm(void *start_code, void *end_code,
 struct vm_area_struct *vm_area_alloc(struct mm_struct *);
 struct vm_area_struct *vm_area_dup(struct vm_area_struct *);
 void vm_area_free(struct vm_area_struct *);
+/* Use only if VMA has no other users */
+void __vm_area_free(struct vm_area_struct *vma);
 
 #ifndef CONFIG_MMU
 extern struct rb_root nommu_region_tree;

diff --git a/kernel/fork.c b/kernel/fork.c
index 6944f4eeb4a38598bac8dccb154e52aa5be86ed3..5edb742df85d85af351d6cc3970101392c3d1891 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -480,7 +480,7 @@ struct vm_area_struct *vm_area_dup(struct vm_area_struct *orig)
        return new;
 }
 
-static void __vm_area_free(struct vm_area_struct *vma)
+void __vm_area_free(struct vm_area_struct *vma)
 {
        free_anon_vma_name(vma);
        kmem_cache_free(vm_area_cachep, vma);

diff --git a/mm/mmap.c b/mm/mmap.c
index 40c71e3345ca75b7babc31194b944ab28d703388..a2bc2d9432b84f1a84eab9c299f7a35e641404c8 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -133,7 +133,7 @@ void unlink_file_vma(struct vm_area_struct *vma)
 /*
  * Close a vm structure and free it.
  */
-static void remove_vma(struct vm_area_struct *vma)
+static void remove_vma(struct vm_area_struct *vma, bool unreachable)
 {
        might_sleep();
        if (vma->vm_ops && vma->vm_ops->close)
@@ -141,7 +141,10 @@ static void remove_vma(struct vm_area_struct *vma)
        if (vma->vm_file)
                fput(vma->vm_file);
        mpol_put(vma_policy(vma));
-       vm_area_free(vma);
+       if (unreachable)
+               __vm_area_free(vma);
+       else
+               vm_area_free(vma);
 }
 
 static inline struct vm_area_struct *vma_prev_limit(struct vma_iterator *vmi,
@@ -2129,7 +2132,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
                if (vma->vm_flags & VM_ACCOUNT)
                        nr_accounted += nrpages;
                vm_stat_account(mm, vma->vm_flags, -nrpages);
-               remove_vma(vma);
+               remove_vma(vma, false);
        }
        vm_unacct_memory(nr_accounted);
        validate_mm(mm);
@@ -3063,7 +3066,7 @@ void exit_mmap(struct mm_struct *mm)
        do {
                if (vma->vm_flags & VM_ACCOUNT)
                        nr_accounted += vma_pages(vma);
-               remove_vma(vma);
+               remove_vma(vma, true);
                count++;
                cond_resched();
        } while ((vma = mas_find(&mas, ULONG_MAX)) != NULL);