mmap: change zeroing of maple tree in __vma_adjust()

Only write to the maple tree if we are not inserting or the insert isn't
going to overwrite the area to clear.  This avoids spanning writes and
node coealescing when unnecessary.

The change requires a custom search for the linked list addition to find
the correct VMA for the prev link.

Link: https://lkml.kernel.org/r/20220404143501.2016403-21-Liam.Howlett@oracle.com
Signed-off-by: Liam R. Howlett <Liam.Howlett@oracle.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/mmap.c b/mm/mmap.c
index 58ff4bdf1adf127800943545482b13e0ea3de9ed..8b923160cd8a566702356f6da59bd86623942026 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -590,10 +590,11 @@ static void vma_link(struct mm_struct *mm, struct vm_area_struct *vma,
  * Helper for vma_adjust() in the split_vma insert case: insert a vma into the
  * mm's list and the mm tree.  It has already been inserted into the interval tree.
  */
-static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma)
+static void __insert_vm_struct(struct mm_struct *mm, struct vm_area_struct *vma,
+                              unsigned long location)
 {
        struct vm_area_struct *prev;
-       MA_STATE(mas, &mm->mm_mt, vma->vm_start, vma->vm_start);
+       MA_STATE(mas, &mm->mm_mt, location, location);
 
        prev = mas_prev(&mas, 0);
        vma_store(mm, vma);
@@ -621,6 +622,7 @@ int __vma_adjust(struct vm_area_struct *vma, unsigned long start,
        bool vma_changed = false;
        long adjust_next = 0;
        int remove_next = 0;
+       unsigned long ll_prev = vma->vm_start; /* linked list prev. */
 
        if (next && !insert) {
                struct vm_area_struct *exporter = NULL, *importer = NULL;
@@ -747,17 +749,27 @@ again:
        }
 
        if (start != vma->vm_start) {
-               if (vma->vm_start < start)
+               if ((vma->vm_start < start) &&
+                   (!insert || (insert->vm_end != start))) {
                        vma_mt_szero(mm, vma->vm_start, start);
-               else
+                       VM_WARN_ON(insert && insert->vm_start > vma->vm_start);
+               } else {
                        vma_changed = true;
+               }
                vma->vm_start = start;
        }
        if (end != vma->vm_end) {
-               if (vma->vm_end > end)
-                       vma_mt_szero(mm, end, vma->vm_end);
-               else
+               if (vma->vm_end > end) {
+                       if (!insert || (insert->vm_start != end)) {
+                               vma_mt_szero(mm, end, vma->vm_end);
+                               VM_WARN_ON(insert &&
+                                          insert->vm_end < vma->vm_end);
+                       } else if (insert->vm_start == end) {
+                               ll_prev = vma->vm_end;
+                       }
+               } else {
                        vma_changed = true;
+               }
                vma->vm_end = end;
                if (!next)
                        mm->highest_vm_end = vm_end_gap(vma);
@@ -792,7 +804,7 @@ again:
                 * us to insert it before dropping the locks
                 * (it may either follow vma or precede it).
                 */
-               __insert_vm_struct(mm, insert);
+               __insert_vm_struct(mm, insert, ll_prev);
        }
 
        if (anon_vma) {