mm/mmap: Linked list fallout

Signed-off-by: Liam R. Howlett <Liam.Howlett@Oracle.com>

diff --git a/mm/mmap.c b/mm/mmap.c
index 2945140df9b80f3802640e961c2f9e8b07f7dee2..1ea06a183c9bdd219adbbfd335c3c04efc8ccafa 100644 (file)
--- a/mm/mmap.c
+++ b/mm/mmap.c
@@ -307,7 +307,7 @@ static void validate_mm(struct mm_struct *mm)
        MA_STATE(mas, &mm->mm_mt, 0, 0);
 
 
-       mas_for_each(mas, vma, ULONG_MAX) {
+       mas_for_each(&mas, vma, ULONG_MAX) {
 #ifdef CONFIG_DEBUG_VM_RB
                struct anon_vma *anon_vma = vma->anon_vma;
                struct anon_vma_chain *avc;
@@ -582,7 +582,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma,
        vma->vm_start = start;
        vma->vm_end = end;
        vma->vm_pgoff = pgoff;
-       /* Note: mas must be pointing to the expanding VMA */
        vma_mas_store(vma, mas);
 
        if (file) {
@@ -2168,7 +2167,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas)
 
        /* Update high watermark before we lower total_vm */
        update_hiwater_vm(mm);
-       mas_for_each(mas, vma, ULONG_MAX) {
+       mas_for_each(mas, vma, -1) {
                long nrpages = vma_pages(vma);
 
                if (vma->vm_flags & VM_ACCOUNT)
@@ -2337,6 +2336,7 @@ int do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
        struct maple_tree mt_detach = MTREE_INIT(mt_detach, MAPLE_ALLOC_RANGE);
        unsigned long max;
        MA_STATE(dst, &mt_detach, start, start);
+       struct ma_state tmp;
        /* we have start < vma->vm_end  */
 
        /*
@@ -2360,25 +2360,31 @@ int do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma,
                if (error)
                        return error;
                prev = vma;
-               vma = vma_next(mm, prev);
+               // Split invalidated node, reset.
                mas->index = start;
                mas_reset(mas);
+               vma = mas_walk(mas);
        } else {
-               prev = vma_prev(mm, vma);
+               tmp = *mas;
+               prev = mas_prev(&tmp, 0);
        }
 
        if (vma->vm_end >= end)
                last = vma;
-       else
-               last = find_vma_intersection(mm, end - 1, end);
+       else {
+               tmp = *mas;
+               last = mas_next(&tmp, -1);
+       }
 
        /* Does it split the last one? */
        if (last && end < last->vm_end) {
                int error = __split_vma(mm, last, end, 1);
                if (error)
                        return error;
-               vma = vma_next(mm, prev);
+               // Split invalidated node, reset.
+               mas->index = start;
                mas_reset(mas);
+               vma = mas_walk(mas);
        }
 
 
@@ -2494,7 +2500,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
        unsigned long max = USER_PGTABLES_CEILING;
        pgoff_t vm_pgoff;
        int error;
-       struct ma_state ma_prev;
+       struct ma_state ma_prev, tmp;
        MA_STATE(mas, &mm->mm_mt, addr, end - 1);
 
        /* Check against address space limit. */
@@ -2526,9 +2532,10 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
                vm_flags |= VM_ACCOUNT;
        }
 
-
-       ma_prev = mas;
+       mas_reset(&mas);
+       mas_set_range(&mas, addr, end - 1);
        if (vm_flags & VM_SPECIAL) {
+               ma_prev = mas;
                prev = mas_prev(&ma_prev, 0);
                goto cannot_expand;
        }
@@ -2536,10 +2543,10 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
        /* Attempt to expand an old mapping */
 
        /* Check next */
-       next = mas_next(&ma_prev, ULONG_MAX);
+       tmp = mas;
+       next = mas_next(&tmp, ULONG_MAX);
        if (next) {
                max = next->vm_start;
-
                if (next->vm_start == end && vma_policy(next) &&
                    can_vma_merge_before(next, vm_flags, NULL, file,
                                         pgoff + pglen, NULL_VM_UFFD_CTX)) {
@@ -2550,18 +2557,20 @@ unsigned long mmap_region(struct file *file, unsigned long addr,
        }
 
        /* Check prev */
+       ma_prev = tmp;
        prev = mas_prev(&ma_prev, 0);
        if (prev && prev->vm_end == addr && !vma_policy(prev) &&
            can_vma_merge_after(prev, vm_flags, NULL, file, pgoff,
                                NULL_VM_UFFD_CTX)) {
                merge_start = prev->vm_start;
                vma = prev;
+               tmp = ma_prev;
                vm_pgoff = prev->vm_pgoff;
        }
 
        /* Actually expand, if possible */
        if (vma &&
-           !vma_expand(&ma_prev, vma, merge_start, merge_end, vm_pgoff, next)) {
+           !vma_expand(&tmp, vma, merge_start, merge_end, vm_pgoff, next)) {
                khugepaged_enter_vma_merge(prev, vm_flags);
                goto expanded;
        }
@@ -2661,6 +2670,10 @@ cannot_expand:
                        goto free_vma;
        }
 
+       // Very likely a shorter walk.
+       mas = ma_prev;
+       mas_set_range(&mas, addr, end - 1);
+       mas_walk(&mas);
        vma_mas_link(mm, vma, &mas);
        /* Once vma denies write, undo our temporary denial count */
        if (file) {
@@ -2704,8 +2717,6 @@ unmap_and_free_vma:
        vma->vm_file = NULL;
        fput(file);
 
-       mas.index = mas.last = addr;
-       vma = mas_walk(&mas);
        /* Undo any partial mapping done by a device driver. */
        unmap_region(mm, vma, &mas, vma->vm_start, vma->vm_end, prev, max);
        charged = 0;
@@ -3149,7 +3160,7 @@ void exit_mmap(struct mm_struct *mm)
         */
        mas_reset(&mas);
        mas_set(&mas, 0);
-       mas_for_each(&mas, vma, ULONG_MAX) {
+       mas_for_each(&mas, vma, -1) {
                if (vma->vm_flags & VM_ACCOUNT)
                        nr_accounted += vma_pages(vma);
                remove_vma(vma);
@@ -3559,6 +3570,7 @@ int mm_take_all_locks(struct mm_struct *mm)
        }
 
        mas_reset(&mas);
+       mas_set(&mas, 0);
        mas_for_each(&mas, vma, ULONG_MAX) {
                if (signal_pending(current))
                        goto out_unlock;
@@ -3568,6 +3580,7 @@ int mm_take_all_locks(struct mm_struct *mm)
        }
 
        mas_reset(&mas);
+       mas_set(&mas, 0);
        mas_for_each(&mas, vma, ULONG_MAX) {
                if (signal_pending(current))
                        goto out_unlock;