kernel/fork: Use maple tree for dup_mmap() during forking

The maple tree was already tracking VMAs in this function by an earlier
commit, but the rbtree iterator was being used to iterate the list.
Change the iterator to use a maple tree native iterator, rcu locking,
and switch to the maple tree advanced API to avoid multiple walks of the
tree during insert operations.

anon_vma_fork() may enter the slow path and cause a schedule() call to
cause rcu issues.  Drop the rcu lock and reacquiring the lock.  There is
no harm in this approach as the mmap_sem is taken for write/read and
held across the schedule() call so the VMAs will not change.

Note that the bulk allocation of nodes is also happening here for
performance reasons.  The node calculations are done internally to the
tree and use the VMA count and assume the worst-case node requirements.
The VM_DONT_COPY flag does not allow for the most efficient copy method
of the tree and so a bulk loading algorithm is used.

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 26b30d53cef8c56ef290070e40380fa1f3dbbc90..5f2a15e702ffe4aabfb6dc773d6476db26202496 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2498,8 +2498,6 @@ extern bool arch_has_descending_max_zone_pfns(void);
 /* nommu.c */
 extern atomic_long_t mmap_pages_allocated;
 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
-/* maple_tree */
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma);
 
 /* interval_tree.c */
 void vma_interval_tree_insert(struct vm_area_struct *node,

diff --git a/include/linux/sched/mm.h b/include/linux/sched/mm.h
index e24b1fe348e3bf87efb3440101fa0762cec88a93..76cab3aea6ab69bc12e437285053e9cc24f4e463 100644 (file)
--- a/include/linux/sched/mm.h
+++ b/include/linux/sched/mm.h
@@ -8,6 +8,7 @@
 #include <linux/mm_types.h>
 #include <linux/gfp.h>
 #include <linux/sync_core.h>
+#include <linux/maple_tree.h>
 
 /*
  * Routines for handling mm_structs
@@ -67,11 +68,13 @@ static inline void mmdrop(struct mm_struct *mm)
  */
 static inline void mmget(struct mm_struct *mm)
 {
+       mt_set_in_rcu(&mm->mm_mt);
        atomic_inc(&mm->mm_users);
 }
 
 static inline bool mmget_not_zero(struct mm_struct *mm)
 {
+       mt_set_in_rcu(&mm->mm_mt);
        return atomic_inc_not_zero(&mm->mm_users);
 }
 

diff --git a/kernel/fork.c b/kernel/fork.c
index df9edc9c8fdd1f60243b7c177cb47dc738f2cb4d..c5d132f9bc7c4e11553f984efdba451a7b81513c 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -477,7 +477,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
        struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
        struct rb_node **rb_link, *rb_parent;
        int retval;
-       unsigned long charge;
+       unsigned long charge = 0;
+       MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
+       MA_STATE(mas, &mm->mm_mt, 0, 0);
        LIST_HEAD(uf);
 
        uprobe_start_dup_mmap();
@@ -511,7 +513,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                goto out;
 
        prev = NULL;
-       for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+
+       retval = mas_entry_count(&mas, oldmm->map_count);
+       if (retval)
+               goto out;
+
+       mas_for_each(&old_mas, mpnt, ULONG_MAX) {
                struct file *file;
 
                if (mpnt->vm_flags & VM_DONTCOPY) {
@@ -525,7 +532,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                 */
                if (fatal_signal_pending(current)) {
                        retval = -EINTR;
-                       goto out;
+                       goto loop_out;
                }
                if (mpnt->vm_flags & VM_ACCOUNT) {
                        unsigned long len = vma_pages(mpnt);
@@ -553,6 +560,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                        tmp->anon_vma = NULL;
                } else if (anon_vma_fork(tmp, mpnt))
                        goto fail_nomem_anon_vma_fork;
+
                tmp->vm_flags &= ~(VM_LOCKED | VM_LOCKONFAULT);
                file = tmp->vm_file;
                if (file) {
@@ -594,7 +602,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                rb_parent = &tmp->vm_rb;
 
                /* Link the vma into the MT */
-               vma_store(mm, tmp);
+               mas.index = tmp->vm_start;
+               mas.last = tmp->vm_end - 1;
+               mas_store(&mas, tmp);
 
                mm->map_count++;
                if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -604,10 +614,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                        tmp->vm_ops->open(tmp);
 
                if (retval)
-                       goto out;
+                       goto loop_out;
        }
        /* a new mm has just been created */
        retval = arch_dup_mmap(oldmm, mm);
+loop_out:
+       mas_destroy(&mas);
 out:
        mmap_write_unlock(mm);
        flush_tlb_mm(oldmm);
@@ -623,7 +635,7 @@ fail_nomem_policy:
 fail_nomem:
        retval = -ENOMEM;
        vm_unacct_memory(charge);
-       goto out;
+       goto loop_out;
 }
 
 static inline int mm_alloc_pgd(struct mm_struct *mm)
@@ -1093,6 +1105,7 @@ static inline void __mmput(struct mm_struct *mm)
 {
        VM_BUG_ON(atomic_read(&mm->mm_users));
 
+       mt_clear_in_rcu(&mm->mm_mt);
        uprobe_clear_state(mm);
        exit_aio(mm);
        ksm_exit(mm);

diff --git a/mm/mmap.c b/mm/mmap.c
index c62142c75e675869970344c90788b08ef6a1867c..e9cdb6cb6172ea021358524715f268564757838c 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -781,10 +781,6 @@ void vma_mt_store(struct mm_struct *mm, struct vm_area_struct *vma)
                GFP_KERNEL);
 }
 
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma) {
-       vma_mt_store(mm, vma);
-}
-
 static void
 __vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
        struct vm_area_struct *prev, struct rb_node **rb_link,