kernel/fork: Use maple tree for dup_mmap() during forking

The maple tree was already tracking VMAs in this function by an earlier
commit, but the rbtree iterator was being used to iterate the list.
Change the iterator to use a maple tree native iterator, rcu locking,
and switch to the maple tree advanced API to avoid multiple walks of the
tree during insert operations.

The maple tree header is now included and so use the maple tree
interface directly.  Drop the unused vma_store() function.

Note that the bulk allocation of nodes is also happening here for
performance reasons.  The node calculations are done internally to the
tree and use the VMA count and assume the worst-case node requirements.
The VM_DONT_COPY flag does not allow for the most efficient copy method
of the tree and so a bulk loading algorithm is used.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index df592bf937f9084f05ed37c5369b44e558be2119..73a52aba448f940cb3b0574d61729c4436853a5e 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2500,8 +2500,6 @@ extern bool arch_has_descending_max_zone_pfns(void);
 /* nommu.c */
 extern atomic_long_t mmap_pages_allocated;
 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
-/* maple_tree */
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma);
 
 /* interval_tree.c */
 void vma_interval_tree_insert(struct vm_area_struct *node,

diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index 5561486fddef7a31675958319a240a9369e2bdd7..2ca7cc6c903bc57b1303168fe3bb96964f7b726a 100644 (file)
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -8,6 +8,7 @@
 #include <linux/mm_types.h>
 #include <linux/gfp.h>
 #include <linux/sync_core.h>
+#include <linux/maple_tree.h>
 
 /*
  * Routines for handling mm_structs
@@ -49,6 +50,8 @@ static inline void mmdrop(struct mm_struct *mm)
                __mmdrop(mm);
 }
 
+void mm_set_in_rcu(struct mm_struct *mm);
+
 /**
  * mmget() - Pin the address space associated with a &struct mm_struct.
  * @mm: The address space to pin.
@@ -67,11 +70,21 @@ static inline void mmdrop(struct mm_struct *mm)
  */
 static inline void mmget(struct mm_struct *mm)
 {
+       if (!mt_in_rcu(&mm->mm_mt))
+               mm_set_in_rcu(mm);
        atomic_inc(&mm->mm_users);
 }
 
 static inline bool mmget_not_zero(struct mm_struct *mm)
 {
+       /*
+        * There is a race below during task tear down that can cause the maple
+        * tree to enter rcu mode with only a single user.  If this race
+        * happens, the result would be that the maple tree nodes would remain
+        * active for an extra RCU read cycle.
+        */
+       if (!mt_in_rcu(&mm->mm_mt))
+               mm_set_in_rcu(mm);
        return atomic_inc_not_zero(&mm->mm_users);
 }
 

diff --git a/kernel/fork.c b/kernel/fork.c
index 3f3f823541ccefd1b5031a44a2821d153adfd979..e0229bbf057d499b42b484e165820e5f1c60c11e 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -492,7 +492,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
        struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
        struct rb_node **rb_link, *rb_parent;
        int retval;
-       unsigned long charge;
+       unsigned long charge = 0;
+       MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
+       MA_STATE(mas, &mm->mm_mt, 0, 0);
        LIST_HEAD(uf);
 
        uprobe_start_dup_mmap();
@@ -526,7 +528,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                goto out;
 
        prev = NULL;
-       for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+
+       retval = mas_entry_count(&mas, oldmm->map_count);
+       if (retval)
+               goto out;
+
+       mas_for_each(&old_mas, mpnt, ULONG_MAX) {
                struct file *file;
 
                if (mpnt->vm_flags & VM_DONTCOPY) {
@@ -540,7 +547,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                 */
                if (fatal_signal_pending(current)) {
                        retval = -EINTR;
-                       goto out;
+                       goto loop_out;
                }
                if (mpnt->vm_flags & VM_ACCOUNT) {
                        unsigned long len = vma_pages(mpnt);
@@ -568,6 +575,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                        tmp->anon_vma = NULL;
                } else if (anon_vma_fork(tmp, mpnt))
                        goto fail_nomem_anon_vma_fork;
+
                tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
                file = tmp->vm_file;
                if (file) {
@@ -606,7 +614,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                rb_parent = &tmp->vm_rb;
 
                /* Link the vma into the MT */
-               vma_store(mm, tmp);
+               mas.index = tmp->vm_start;
+               mas.last = tmp->vm_end - 1;
+               mas_store(&mas, tmp);
 
                mm->map_count++;
                if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -616,10 +626,13 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                        tmp->vm_ops->open(tmp);
 
                if (retval)
-                       goto out;
+                       goto loop_out;
+
        }
        /* a new mm has just been created */
        retval = arch_dup_mmap(oldmm, mm);
+loop_out:
+       mas_destroy(&mas);
 out:
        mmap_write_unlock(mm);
        flush_tlb_mm(oldmm);
@@ -635,7 +648,7 @@ fail_nomem_policy:
 fail_nomem:
        retval = -ENOMEM;
        vm_unacct_memory(charge);
-       goto out;
+       goto loop_out;
 }
 
 static inline int mm_alloc_pgd(struct mm_struct *mm)
@@ -1113,6 +1126,7 @@ static inline void __mmput(struct mm_struct *mm)
 {
        VM_BUG_ON(atomic_read(&mm->mm_users));
 
+       mt_clear_in_rcu(&mm->mm_mt);
        uprobe_clear_state(mm);
        exit_aio(mm);
        ksm_exit(mm);

diff --git a/mm/mmap.c b/mm/mmap.c
index a5fae5c910ecd03ff4775e254a41427c74c92196..637f08c9ecfa0c5862ce2ab1a58d67e46ab533b4 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -309,6 +309,15 @@ static inline unsigned long vma_compute_gap(struct vm_area_struct *vma)
        return gap;
 }
 
+void mm_set_in_rcu(struct mm_struct *mm)
+{
+       if (!mt_in_rcu(&mm->mm_mt))
+               return;
+       mmap_write_lock(mm);
+       mm->mm_mt.ma_flags |= MT_FLAGS_USE_RCU;
+       mmap_write_unlock(mm);
+}
+
 #ifdef CONFIG_DEBUG_VM_RB
 static unsigned long vma_compute_subtree_gap(struct vm_area_struct *vma)
 {
@@ -778,10 +787,6 @@ void vma_mt_store(struct mm_struct *mm, struct vm_area_struct *vma)
        mas_store_gfp(&mas, vma, GFP_KERNEL);
 }
 
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma) {
-       vma_mt_store(mm, vma);
-}
-
 static void
 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
        struct vm_area_struct *prev, struct rb_node **rb_link,