mm/mmap: Change do_brk_flags() to expand existing VMA when possible.

Avoid allocating a new VMA when it is not necessary.  Expand the
existing VMA instead.  This avoids unnecessary tree manipulations.

kvm benchmark before
$ ./mmtests/work/sources/wis-0-installed/brk1_threads
testcase:brk increase/decrease of one page
warmup
min:561754 max:561754 total:561754
min:578322 max:578322 total:578322
min:581282 max:581282 total:581282
min:572182 max:572182 total:572182
min:572572 max:572572 total:572572
min:580408 max:580408 total:580408
measurement
min:578786 max:578786 total:578786
min:574780 max:574780 total:574780
min:577166 max:577166 total:577166
min:581240 max:581240 total:581240
min:580700 max:580700 total:580700
min:584612 max:584612 total:584612

benchmark after
$ ./mmtests/work/sources/wis-0-installed/brk1_threads
testcase:brk increase/decrease of one page
warmup
min:886150 max:886150 total:886150
min:901488 max:901488 total:901488
min:897594 max:897594 total:897594
min:897422 max:897422 total:897422
min:900700 max:900700 total:900700
min:900714 max:900714 total:900714
measurement
min:903016 max:903016 total:903016
min:901224 max:901224 total:901224
min:902626 max:902626 total:902626
min:900518 max:900518 total:900518
min:900630 max:900630 total:900630
min:901432 max:901432 total:901432

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/mm/mmap.c b/mm/mmap.c
index 129f4adc3af5e37631b77a907ec6b356377ed69c..cffa49eaf2b83e2e20fb4d2c25e3faae96cef7d0 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -184,14 +184,15 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma)
        return next;
 }
 
-static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags,
-               struct list_head *uf);
+static int do_brk_flags(unsigned long addr, unsigned long request,
+                       struct vm_area_struct *vma, unsigned long flags,
+                       struct list_head *uf);
 SYSCALL_DEFINE1(brk, unsigned long, brk)
 {
        unsigned long retval;
        unsigned long newbrk, oldbrk, origbrk;
        struct mm_struct *mm = current->mm;
-       struct vm_area_struct *next;
+       struct vm_area_struct *vma_brk, *next;
        unsigned long min_brk;
        bool populate;
        bool downgraded = false;
@@ -259,12 +260,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk)
        }
 
        /* Check against existing mmap mappings. */
-       next = find_vma(mm, oldbrk);
+       next = find_vma_prev(mm, oldbrk, &vma_brk);
        if (next && newbrk + PAGE_SIZE > vm_start_gap(next))
                goto out;
 
        /* Ok, looks good - let it rip. */
-       if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0)
+       if (do_brk_flags(oldbrk, newbrk-oldbrk, vma_brk, 0, &uf) < 0)
                goto out;
        mm->brk = brk;
 
@@ -568,6 +569,7 @@ static void __vma_link_file(struct vm_area_struct *vma)
                flush_dcache_mmap_unlock(mapping);
        }
 }
+
 /* Private
  * vma_mt_erase() - erase a VMA entry from the maple tree.
  *
@@ -606,6 +608,22 @@ static inline void vma_mt_store(struct mm_struct *mm, struct vm_area_struct *vma
                GFP_KERNEL);
 }
 
+void vma_mt_modify(struct vm_area_struct *vma, unsigned long new_start,
+               unsigned long new_end)
+{
+       // Shrinking front.
+       if (vma->vm_start < new_start)
+               vma_mt_szero(vma->vm_mm, vma->vm_start, new_start);
+
+       // Shrinking back.
+       if (vma->vm_end > new_end)
+               vma_mt_szero(vma->vm_mm, new_end, vma->vm_end);
+
+       vma->vm_start = new_start;
+       vma->vm_end = new_end;
+       vma_mt_store(vma->vm_mm, vma);
+}
+
 static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
                        struct vm_area_struct *prev)
 {
@@ -2063,8 +2081,7 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len,
 EXPORT_SYMBOL(get_unmapped_area);
 
 /**
- * find_vma() - Find the VMA for a given address, or the next vma.  May return
- * NULL in the case of no vma at addr or above
+ * find_vma() - Find the VMA for a given address, or the next vma.
  * @mm The mm_struct to check
  * @addr: The address
  *
@@ -2813,15 +2830,22 @@ out:
 }
 
 /*
- *  this is really a simplified "do_mmap".  it only handles
- *  anonymous maps.  eventually we may be able to do some
- *  brk-specific accounting here.
+ * do_brk_flags() - Increase the brk vma if the flags match.
+ * @addr: The start address
+ * @len: The length of the increase
+ * @vma: The vma,
+ * @flags: The VMA Flags
+ *
+ * Extend the brk VMA from addr to addr + len.  If the VMA is NULL or the flags
+ * do not match then create a new anonymous VMA.  Eventually we may be able to
+ * do some brk-specific accounting here.
  */
 static int do_brk_flags(unsigned long addr, unsigned long len,
-                       unsigned long flags, struct list_head *uf)
+                       struct vm_area_struct *vma, unsigned long flags,
+                       struct list_head *uf)
 {
        struct mm_struct *mm = current->mm;
-       struct vm_area_struct *vma, *prev;
+       struct vm_area_struct *prev = NULL;
        pgoff_t pgoff = addr >> PAGE_SHIFT;
        int error;
        unsigned long mapped_addr;
@@ -2840,11 +2864,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len,
        if (error)
                return error;
 
-       /* Clear old maps, set up prev and uf */
-       if (munmap_vma_range(mm, addr, len, &prev, uf))
-               return -ENOMEM;
-
-       /* Check against address space limits *after* clearing old maps... */
+       /* Check against address space limits by the changed size */
        if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT))
                return -ENOMEM;
 
@@ -2854,15 +2874,15 @@ static int do_brk_flags(unsigned long addr, unsigned long len,
        if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT))
                return -ENOMEM;
 
-       /* Can we just expand an old private anonymous mapping? */
-       vma = vma_merge(mm, prev, addr, addr + len, flags,
-                       NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX);
-       if (vma)
+       if (vma && flags == vma->vm_flags) {
+               vma_mt_modify(vma, vma->vm_start, addr+len);
                goto out;
+       }
 
        /*
         * create a vma struct for an anonymous mapping
         */
+       prev = vma;
        vma = vm_area_alloc(mm);
        if (!vma) {
                vm_unacct_memory(len >> PAGE_SHIFT);
@@ -2875,6 +2895,8 @@ static int do_brk_flags(unsigned long addr, unsigned long len,
        vma->vm_pgoff = pgoff;
        vma->vm_flags = flags;
        vma->vm_page_prot = vm_get_page_prot(flags);
+       if (!prev)
+               find_vma_prev(mm, addr, &prev);
        vma_link(mm, vma, prev);
 out:
        perf_event_mmap(vma);
@@ -2904,7 +2926,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags)
        if (mmap_write_lock_killable(mm))
                return -EINTR;
 
-       ret = do_brk_flags(addr, len, flags, &uf);
+       ret = do_brk_flags(addr, len, NULL, flags, &uf);
        populate = ((mm->def_flags & VM_LOCKED) != 0);
        mmap_write_unlock(mm);
        userfaultfd_unmap_complete(mm, &uf);