From: Liam R. Howlett Date: Mon, 21 Sep 2020 14:47:34 +0000 (-0400) Subject: mm/mmap: Change do_brk_flags() to expand existing VMA when possible. X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=b91a6360cf8f29ea50b5abbc44e4f54f1e541ee1;p=users%2Fjedix%2Flinux-maple.git mm/mmap: Change do_brk_flags() to expand existing VMA when possible. Avoid allocating a new VMA when it is not necessary. Expand the existing VMA instead. This avoids unnecessary tree manipulations. kvm benchmark before $ ./mmtests/work/sources/wis-0-installed/brk1_threads testcase:brk increase/decrease of one page warmup min:561754 max:561754 total:561754 min:578322 max:578322 total:578322 min:581282 max:581282 total:581282 min:572182 max:572182 total:572182 min:572572 max:572572 total:572572 min:580408 max:580408 total:580408 measurement min:578786 max:578786 total:578786 min:574780 max:574780 total:574780 min:577166 max:577166 total:577166 min:581240 max:581240 total:581240 min:580700 max:580700 total:580700 min:584612 max:584612 total:584612 benchmark after $ ./mmtests/work/sources/wis-0-installed/brk1_threads testcase:brk increase/decrease of one page warmup min:886150 max:886150 total:886150 min:901488 max:901488 total:901488 min:897594 max:897594 total:897594 min:897422 max:897422 total:897422 min:900700 max:900700 total:900700 min:900714 max:900714 total:900714 measurement min:903016 max:903016 total:903016 min:901224 max:901224 total:901224 min:902626 max:902626 total:902626 min:900518 max:900518 total:900518 min:900630 max:900630 total:900630 min:901432 max:901432 total:901432 Signed-off-by: Liam R. Howlett --- diff --git a/mm/mmap.c b/mm/mmap.c index 129f4adc3af5..cffa49eaf2b8 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -184,14 +184,15 @@ static struct vm_area_struct *remove_vma(struct vm_area_struct *vma) return next; } -static int do_brk_flags(unsigned long addr, unsigned long request, unsigned long flags, - struct list_head *uf); +static int do_brk_flags(unsigned long addr, unsigned long request, + struct vm_area_struct *vma, unsigned long flags, + struct list_head *uf); SYSCALL_DEFINE1(brk, unsigned long, brk) { unsigned long retval; unsigned long newbrk, oldbrk, origbrk; struct mm_struct *mm = current->mm; - struct vm_area_struct *next; + struct vm_area_struct *vma_brk, *next; unsigned long min_brk; bool populate; bool downgraded = false; @@ -259,12 +260,12 @@ SYSCALL_DEFINE1(brk, unsigned long, brk) } /* Check against existing mmap mappings. */ - next = find_vma(mm, oldbrk); + next = find_vma_prev(mm, oldbrk, &vma_brk); if (next && newbrk + PAGE_SIZE > vm_start_gap(next)) goto out; /* Ok, looks good - let it rip. */ - if (do_brk_flags(oldbrk, newbrk-oldbrk, 0, &uf) < 0) + if (do_brk_flags(oldbrk, newbrk-oldbrk, vma_brk, 0, &uf) < 0) goto out; mm->brk = brk; @@ -568,6 +569,7 @@ static void __vma_link_file(struct vm_area_struct *vma) flush_dcache_mmap_unlock(mapping); } } + /* Private * vma_mt_erase() - erase a VMA entry from the maple tree. * @@ -606,6 +608,22 @@ static inline void vma_mt_store(struct mm_struct *mm, struct vm_area_struct *vma GFP_KERNEL); } +void vma_mt_modify(struct vm_area_struct *vma, unsigned long new_start, + unsigned long new_end) +{ + // Shrinking front. + if (vma->vm_start < new_start) + vma_mt_szero(vma->vm_mm, vma->vm_start, new_start); + + // Shrinking back. + if (vma->vm_end > new_end) + vma_mt_szero(vma->vm_mm, new_end, vma->vm_end); + + vma->vm_start = new_start; + vma->vm_end = new_end; + vma_mt_store(vma->vm_mm, vma); +} + static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma, struct vm_area_struct *prev) { @@ -2063,8 +2081,7 @@ get_unmapped_area(struct file *file, unsigned long addr, unsigned long len, EXPORT_SYMBOL(get_unmapped_area); /** - * find_vma() - Find the VMA for a given address, or the next vma. May return - * NULL in the case of no vma at addr or above + * find_vma() - Find the VMA for a given address, or the next vma. * @mm The mm_struct to check * @addr: The address * @@ -2813,15 +2830,22 @@ out: } /* - * this is really a simplified "do_mmap". it only handles - * anonymous maps. eventually we may be able to do some - * brk-specific accounting here. + * do_brk_flags() - Increase the brk vma if the flags match. + * @addr: The start address + * @len: The length of the increase + * @vma: The vma, + * @flags: The VMA Flags + * + * Extend the brk VMA from addr to addr + len. If the VMA is NULL or the flags + * do not match then create a new anonymous VMA. Eventually we may be able to + * do some brk-specific accounting here. */ static int do_brk_flags(unsigned long addr, unsigned long len, - unsigned long flags, struct list_head *uf) + struct vm_area_struct *vma, unsigned long flags, + struct list_head *uf) { struct mm_struct *mm = current->mm; - struct vm_area_struct *vma, *prev; + struct vm_area_struct *prev = NULL; pgoff_t pgoff = addr >> PAGE_SHIFT; int error; unsigned long mapped_addr; @@ -2840,11 +2864,7 @@ static int do_brk_flags(unsigned long addr, unsigned long len, if (error) return error; - /* Clear old maps, set up prev and uf */ - if (munmap_vma_range(mm, addr, len, &prev, uf)) - return -ENOMEM; - - /* Check against address space limits *after* clearing old maps... */ + /* Check against address space limits by the changed size */ if (!may_expand_vm(mm, flags, len >> PAGE_SHIFT)) return -ENOMEM; @@ -2854,15 +2874,15 @@ static int do_brk_flags(unsigned long addr, unsigned long len, if (security_vm_enough_memory_mm(mm, len >> PAGE_SHIFT)) return -ENOMEM; - /* Can we just expand an old private anonymous mapping? */ - vma = vma_merge(mm, prev, addr, addr + len, flags, - NULL, NULL, pgoff, NULL, NULL_VM_UFFD_CTX); - if (vma) + if (vma && flags == vma->vm_flags) { + vma_mt_modify(vma, vma->vm_start, addr+len); goto out; + } /* * create a vma struct for an anonymous mapping */ + prev = vma; vma = vm_area_alloc(mm); if (!vma) { vm_unacct_memory(len >> PAGE_SHIFT); @@ -2875,6 +2895,8 @@ static int do_brk_flags(unsigned long addr, unsigned long len, vma->vm_pgoff = pgoff; vma->vm_flags = flags; vma->vm_page_prot = vm_get_page_prot(flags); + if (!prev) + find_vma_prev(mm, addr, &prev); vma_link(mm, vma, prev); out: perf_event_mmap(vma); @@ -2904,7 +2926,7 @@ int vm_brk_flags(unsigned long addr, unsigned long request, unsigned long flags) if (mmap_write_lock_killable(mm)) return -EINTR; - ret = do_brk_flags(addr, len, flags, &uf); + ret = do_brk_flags(addr, len, NULL, flags, &uf); populate = ((mm->def_flags & VM_LOCKED) != 0); mmap_write_unlock(mm); userfaultfd_unmap_complete(mm, &uf);