kernel/fork: Use maple tree for dup_mmap() during forking

The maple tree was already tracking VMAs in this function by an earlier
commit, but the rbtree iterator was being used to iterate the list.
Change the iterator to use a maple tree native iterator and switch to
the maple tree advanced API to avoid multiple walks of the tree during
insert operations.  Unexport the now-unused vma_store() function.

For performance reasons we bulk allocate the maple tree nodes.  The node
calculations are done internally to the tree and use the VMA count and
assume the worst-case node requirements.  The VM_DONT_COPY flag does
not allow for the most efficient copy method of the tree and so a bulk
loading algorithm is used.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/include/linux/mm.h b/include/linux/mm.h
index 5521bf12fbf1bc8dbda835b5ae210c134798811a..e39462d5bb7c5906a03eeb29a01d7ff9878c628d 100644 (file)
--- a/include/linux/mm.h
+++ b/include/linux/mm.h
@@ -2609,8 +2609,6 @@ extern bool arch_has_descending_max_zone_pfns(void);
 /* nommu.c */
 extern atomic_long_t mmap_pages_allocated;
 extern int nommu_shrink_inode_mappings(struct inode *, size_t, size_t);
-/* mmap.c */
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma);
 
 /* interval_tree.c */
 void vma_interval_tree_insert(struct vm_area_struct *node,

diff --git a/kernel/fork.c b/kernel/fork.c
index 51a7971651ef1540b8040ac66212f0c50dff3382..d7e2e581bdfd9c4a75c40a02d6b6bdf840ddedb8 100644 (file)
--- a/kernel/fork.c
+++ b/kernel/fork.c
@@ -494,7 +494,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
        struct vm_area_struct *mpnt, *tmp, *prev, **pprev;
        struct rb_node **rb_link, *rb_parent;
        int retval;
-       unsigned long charge;
+       unsigned long charge = 0;
+       MA_STATE(old_mas, &oldmm->mm_mt, 0, 0);
+       MA_STATE(mas, &mm->mm_mt, 0, 0);
        LIST_HEAD(uf);
 
        uprobe_start_dup_mmap();
@@ -528,7 +530,12 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                goto out;
 
        prev = NULL;
-       for (mpnt = oldmm->mmap; mpnt; mpnt = mpnt->vm_next) {
+
+       retval = mas_expected_entries(&mas, oldmm->map_count);
+       if (retval)
+               goto out;
+
+       mas_for_each(&old_mas, mpnt, ULONG_MAX) {
                struct file *file;
 
                if (mpnt->vm_flags & VM_DONTCOPY) {
@@ -542,7 +549,7 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                 */
                if (fatal_signal_pending(current)) {
                        retval = -EINTR;
-                       goto out;
+                       goto loop_out;
                }
                if (mpnt->vm_flags & VM_ACCOUNT) {
                        unsigned long len = vma_pages(mpnt);
@@ -608,7 +615,9 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                rb_parent = &tmp->vm_rb;
 
                /* Link the vma into the MT */
-               vma_store(mm, tmp);
+               mas.index = tmp->vm_start;
+               mas.last = tmp->vm_end - 1;
+               mas_store(&mas, tmp);
 
                mm->map_count++;
                if (!(tmp->vm_flags & VM_WIPEONFORK))
@@ -618,10 +627,13 @@ static __latent_entropy int dup_mmap(struct mm_struct *mm,
                        tmp->vm_ops->open(tmp);
 
                if (retval)
-                       goto out;
+                       goto loop_out;
+
        }
        /* a new mm has just been created */
        retval = arch_dup_mmap(oldmm, mm);
+loop_out:
+       mas_destroy(&mas);
 out:
        mmap_write_unlock(mm);
        flush_tlb_mm(oldmm);
@@ -637,7 +649,7 @@ fail_nomem_policy:
 fail_nomem:
        retval = -ENOMEM;
        vm_unacct_memory(charge);
-       goto out;
+       goto loop_out;
 }
 
 static inline int mm_alloc_pgd(struct mm_struct *mm)

diff --git a/mm/mmap.c b/mm/mmap.c
index 2d93dc632eec3fef7509445637d0b6d6517f442f..8bd52db4be9d5c687345442a349bb06be95f7dfc 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -751,7 +751,7 @@ static inline void vma_mt_szero(struct mm_struct *mm, unsigned long start,
  * @mm: The struct_mm
  * @vma: The vm_area_struct to store in the maple tree.
  */
-void vma_store(struct mm_struct *mm, struct vm_area_struct *vma)
+static void vma_store(struct mm_struct *mm, struct vm_area_struct *vma)
 {
        MA_STATE(mas, &mm->mm_mt, 0, 0);