From 855b737f114df33720445c82bd548a9fd97cc6a7 Mon Sep 17 00:00:00 2001 From: "Liam R. Howlett" Date: Fri, 18 Dec 2020 13:15:03 -0500 Subject: [PATCH] mm/mmap: Linked list fallout Signed-off-by: Liam R. Howlett --- mm/mmap.c | 47 ++++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/mm/mmap.c b/mm/mmap.c index 2945140df9b8..1ea06a183c9b 100644 --- a/mm/mmap.c +++ b/mm/mmap.c @@ -307,7 +307,7 @@ static void validate_mm(struct mm_struct *mm) MA_STATE(mas, &mm->mm_mt, 0, 0); - mas_for_each(mas, vma, ULONG_MAX) { + mas_for_each(&mas, vma, ULONG_MAX) { #ifdef CONFIG_DEBUG_VM_RB struct anon_vma *anon_vma = vma->anon_vma; struct anon_vma_chain *avc; @@ -582,7 +582,6 @@ inline int vma_expand(struct ma_state *mas, struct vm_area_struct *vma, vma->vm_start = start; vma->vm_end = end; vma->vm_pgoff = pgoff; - /* Note: mas must be pointing to the expanding VMA */ vma_mas_store(vma, mas); if (file) { @@ -2168,7 +2167,7 @@ static inline void remove_mt(struct mm_struct *mm, struct ma_state *mas) /* Update high watermark before we lower total_vm */ update_hiwater_vm(mm); - mas_for_each(mas, vma, ULONG_MAX) { + mas_for_each(mas, vma, -1) { long nrpages = vma_pages(vma); if (vma->vm_flags & VM_ACCOUNT) @@ -2337,6 +2336,7 @@ int do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, struct maple_tree mt_detach = MTREE_INIT(mt_detach, MAPLE_ALLOC_RANGE); unsigned long max; MA_STATE(dst, &mt_detach, start, start); + struct ma_state tmp; /* we have start < vma->vm_end */ /* @@ -2360,25 +2360,31 @@ int do_mas_align_munmap(struct ma_state *mas, struct vm_area_struct *vma, if (error) return error; prev = vma; - vma = vma_next(mm, prev); + // Split invalidated node, reset. mas->index = start; mas_reset(mas); + vma = mas_walk(mas); } else { - prev = vma_prev(mm, vma); + tmp = *mas; + prev = mas_prev(&tmp, 0); } if (vma->vm_end >= end) last = vma; - else - last = find_vma_intersection(mm, end - 1, end); + else { + tmp = *mas; + last = mas_next(&tmp, -1); + } /* Does it split the last one? */ if (last && end < last->vm_end) { int error = __split_vma(mm, last, end, 1); if (error) return error; - vma = vma_next(mm, prev); + // Split invalidated node, reset. + mas->index = start; mas_reset(mas); + vma = mas_walk(mas); } @@ -2494,7 +2500,7 @@ unsigned long mmap_region(struct file *file, unsigned long addr, unsigned long max = USER_PGTABLES_CEILING; pgoff_t vm_pgoff; int error; - struct ma_state ma_prev; + struct ma_state ma_prev, tmp; MA_STATE(mas, &mm->mm_mt, addr, end - 1); /* Check against address space limit. */ @@ -2526,9 +2532,10 @@ unsigned long mmap_region(struct file *file, unsigned long addr, vm_flags |= VM_ACCOUNT; } - - ma_prev = mas; + mas_reset(&mas); + mas_set_range(&mas, addr, end - 1); if (vm_flags & VM_SPECIAL) { + ma_prev = mas; prev = mas_prev(&ma_prev, 0); goto cannot_expand; } @@ -2536,10 +2543,10 @@ unsigned long mmap_region(struct file *file, unsigned long addr, /* Attempt to expand an old mapping */ /* Check next */ - next = mas_next(&ma_prev, ULONG_MAX); + tmp = mas; + next = mas_next(&tmp, ULONG_MAX); if (next) { max = next->vm_start; - if (next->vm_start == end && vma_policy(next) && can_vma_merge_before(next, vm_flags, NULL, file, pgoff + pglen, NULL_VM_UFFD_CTX)) { @@ -2550,18 +2557,20 @@ unsigned long mmap_region(struct file *file, unsigned long addr, } /* Check prev */ + ma_prev = tmp; prev = mas_prev(&ma_prev, 0); if (prev && prev->vm_end == addr && !vma_policy(prev) && can_vma_merge_after(prev, vm_flags, NULL, file, pgoff, NULL_VM_UFFD_CTX)) { merge_start = prev->vm_start; vma = prev; + tmp = ma_prev; vm_pgoff = prev->vm_pgoff; } /* Actually expand, if possible */ if (vma && - !vma_expand(&ma_prev, vma, merge_start, merge_end, vm_pgoff, next)) { + !vma_expand(&tmp, vma, merge_start, merge_end, vm_pgoff, next)) { khugepaged_enter_vma_merge(prev, vm_flags); goto expanded; } @@ -2661,6 +2670,10 @@ cannot_expand: goto free_vma; } + // Very likely a shorter walk. + mas = ma_prev; + mas_set_range(&mas, addr, end - 1); + mas_walk(&mas); vma_mas_link(mm, vma, &mas); /* Once vma denies write, undo our temporary denial count */ if (file) { @@ -2704,8 +2717,6 @@ unmap_and_free_vma: vma->vm_file = NULL; fput(file); - mas.index = mas.last = addr; - vma = mas_walk(&mas); /* Undo any partial mapping done by a device driver. */ unmap_region(mm, vma, &mas, vma->vm_start, vma->vm_end, prev, max); charged = 0; @@ -3149,7 +3160,7 @@ void exit_mmap(struct mm_struct *mm) */ mas_reset(&mas); mas_set(&mas, 0); - mas_for_each(&mas, vma, ULONG_MAX) { + mas_for_each(&mas, vma, -1) { if (vma->vm_flags & VM_ACCOUNT) nr_accounted += vma_pages(vma); remove_vma(vma); @@ -3559,6 +3570,7 @@ int mm_take_all_locks(struct mm_struct *mm) } mas_reset(&mas); + mas_set(&mas, 0); mas_for_each(&mas, vma, ULONG_MAX) { if (signal_pending(current)) goto out_unlock; @@ -3568,6 +3580,7 @@ int mm_take_all_locks(struct mm_struct *mm) } mas_reset(&mas); + mas_set(&mas, 0); mas_for_each(&mas, vma, ULONG_MAX) { if (signal_pending(current)) goto out_unlock; -- 2.50.1